[ Mar 7 22:43:37 Enabled. ] [ Mar 7 22:43:37 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ] [ Mar 7 22:43:37 Method "start" exited with status 0. ] [ Mar 7 22:43:37 Rereading configuration. ] [ Mar 7 22:43:37 No 'refresh' method defined. Treating as :true. ] note: configured to log to "/dev/stdout" {"msg":"finding links [\"net0\", \"net1\"]","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:37.827956973Z","hostname":"buskin","pid":1661,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:247"} {"msg":"Starting mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:37.882129149Z","hostname":"buskin","pid":1661,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:254","addr-objs":"[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]"} {"msg":"Importing mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:37.88216348Z","hostname":"buskin","pid":1661,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:42","path":"/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"} {"msg":"Setting mg-ddm interfaces","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:37.992592095Z","hostname":"buskin","pid":1661,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:50","interfaces":"(\"net0/ll\" \"net1/ll\")"} {"msg":"Enabling mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:38.001622597Z","hostname":"buskin","pid":1661,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:56"} {"msg":"Ensuring zfs key directory exists","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:38.009744722Z","hostname":"buskin","pid":1661,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:264","path":"/var/run/oxide/"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:38.616101216Z","hostname":"buskin","pid":1661,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:91","prefix":"Ipv6Prefix { addr: fdb0:18c0:4d0d:9fb2::, len: 64 }"} {"msg":"Starting KeyManager","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:38.76158244Z","hostname":"buskin","pid":1661,"file":"sled-agent/src/long_running_tasks.rs:117"} {"msg":"Starting StorageManager","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:38.761616891Z","hostname":"buskin","pid":1661,"file":"sled-agent/src/long_running_tasks.rs:129"} {"msg":"Starting StorageMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:38.761708494Z","hostname":"buskin","pid":1661,"file":"sled-agent/src/long_running_tasks.rs:141"} {"msg":"Starting HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:38.761815677Z","hostname":"buskin","pid":1661,"file":"sled-agent/src/long_running_tasks.rs:162","sled_mode":"Scrimlet { asic: SoftNpuZone }"} {"msg":"DumpSetup poll loop started.","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:38.761825617Z","hostname":"buskin","pid":1661,"component":"DumpSetup-archival","file":"sled-agent/src/dump_setup.rs:268"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:38.761843428Z","hostname":"buskin","pid":1661,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:776"} {"msg":"Creating HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:38.761853918Z","hostname":"buskin","pid":1661,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:609"} {"msg":"Generated i86pc baseboard Pc { identifier: \"buskin\", model: \"i86pc\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:38.76490999Z","hostname":"buskin","pid":1661,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:535"} {"msg":"Device is not a Gimlet (i86pc), proceeding with null hardware view","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:38.764924501Z","hostname":"buskin","pid":1661,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:658"} {"msg":"Starting HardwareMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:38.764934951Z","hostname":"buskin","pid":1661,"file":"sled-agent/src/long_running_tasks.rs:176"} {"msg":"Monitoring for hardware updates","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:38.764941022Z","hostname":"buskin","pid":1661,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:716"} {"msg":"Upserting synthetic zpool to Storage Manager: oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:38.764960342Z","hostname":"buskin","pid":1661,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:38.764971523Z","hostname":"buskin","pid":1661,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:38.764978683Z","hostname":"buskin","pid":1661,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:38.764982553Z","hostname":"buskin","pid":1661,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:38.764988603Z","hostname":"buskin","pid":1661,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:38.764992363Z","hostname":"buskin","pid":1661,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:38.764997943Z","hostname":"buskin","pid":1661,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:38.765002593Z","hostname":"buskin","pid":1661,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:38.765006264Z","hostname":"buskin","pid":1661,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxp_31bd71cd-4736-4a12-a387-9b74b050396f","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:38.765011784Z","hostname":"buskin","pid":1661,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:38.765016494Z","hostname":"buskin","pid":1661,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Waiting for boot disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:38.765020034Z","hostname":"buskin","pid":1661,"file":"sled-agent/src/long_running_tasks.rs:88"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:38.765023774Z","hostname":"buskin","pid":1661,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:243","underlay_network_info":"None"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: a462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal } }))","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:38.765078616Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Channel capacity decreased","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:38.949350791Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:326","current":"245","previous":"256"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: b462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal } }))","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:38.949396502Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Found boot disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:38.951137025Z","hostname":"buskin","pid":1661,"file":"sled-agent/src/long_running_tasks.rs:90"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } }))","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.117392996Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }))","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.117435147Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }))","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.117448317Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }))","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.117454158Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }))","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.117462658Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } }))","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.117467888Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } }))","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.117480238Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }))","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.117486219Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } }))","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.117491549Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received DisksChanged({})","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.117499259Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Channel capacity increased","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.117504019Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:336","current":"255","previous":"245"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.1175202Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.117611182Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:111","resources":"StorageResources { disks: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: a462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal } }), Pool { name: ZpoolName { id: a462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal }, info: ZpoolInfo { name: \"oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b\", size: 20937965568, allocated: 508416, free: 20937457152, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: b462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal } }), Pool { name: ZpoolName { id: b462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal }, info: ZpoolInfo { name: \"oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b\", size: 20937965568, allocated: 500736, free: 20937464832, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" } })} }"} {"msg":"Starting Bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.117653414Z","hostname":"buskin","pid":1661,"file":"sled-agent/src/long_running_tasks.rs:200"} {"msg":"No ledger in /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.117725396Z","hostname":"buskin","pid":1661,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:187"} {"msg":"No ledger in /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.117744906Z","hostname":"buskin","pid":1661,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:187"} {"msg":"No ledger found. Loading Fsm::State as Uninitialized","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.117750287Z","hostname":"buskin","pid":1661,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:83"} {"msg":"No ledger in /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.117769447Z","hostname":"buskin","pid":1661,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:187"} {"msg":"No ledger in /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.117774607Z","hostname":"buskin","pid":1661,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:187"} {"msg":"No ledger found for network config","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.117779307Z","hostname":"buskin","pid":1661,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:146"} {"msg":"Starting Bootstore DDMD poller","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.117815119Z","hostname":"buskin","pid":1661,"file":"sled-agent/src/long_running_tasks.rs:205"} {"msg":"Starting ZoneBundler related tasks","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.117821989Z","hostname":"buskin","pid":1661,"file":"sled-agent/src/long_running_tasks.rs:220"} {"msg":"Creating ServiceManager","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.132719459Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:697"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.13275829Z","hostname":"buskin","pid":1661,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:288","time_to_next_cleanup":"599.999895676s","next_cleanup":"Instant { tv_sec: 1679, tv_nsec: 817549917 }"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.132794201Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:3494"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.132804321Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"No ledger in /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.132865683Z","hostname":"buskin","pid":1661,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:187"} {"msg":"No ledger in /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.132874834Z","hostname":"buskin","pid":1661,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:187"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.136153242Z","hostname":"buskin","pid":1661,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:195"} {"msg":"Started listening","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.136248345Z","hostname":"buskin","pid":1661,"component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:44","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Injecting RSS configuration: RackInitializeRequest {\n trust_quorum_peers: None,\n bootstrap_discovery: OnlyOurs,\n ntp_servers: [\n \"0.pool.ntp.org\",\n ],\n dns_servers: [\n 1.1.1.1,\n 9.9.9.9,\n ],\n internal_services_ip_pool_ranges: [\n V4(\n Ipv4Range {\n first: 10.151.1.164,\n last: 10.151.1.173,\n },\n ),\n ],\n external_dns_ips: [\n 10.151.1.164,\n 10.151.1.165,\n ],\n external_dns_zone_name: \"oxide.test\",\n external_certificates: \"\",\n recovery_silo: RecoverySiloConfig {\n silo_name: Name(\n \"recovery\",\n ),\n user_name: UserId(\n \"recovery\",\n ),\n user_password_hash: NewPasswordHash(\n PasswordHashString {\n string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n encoding: B64,\n },\n ),\n },\n rack_network_config: RackNetworkConfigV1 {\n rack_subnet: Ipv6Network {\n addr: fd00:1122:3344:100::,\n prefix: 56,\n },\n infra_ip_first: 192.168.1.30,\n infra_ip_last: 192.168.1.30,\n ports: [\n PortConfigV1 {\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Network {\n addr: 0.0.0.0,\n prefix: 0,\n },\n ),\n nexthop: 10.151.1.100,\n },\n ],\n addresses: [\n V4(\n Ipv4Network {\n addr: 10.151.1.174,\n prefix: 24,\n },\n ),\n ],\n switch: Switch0,\n port: \"qsfp0\",\n uplink_port_speed: Speed40G,\n uplink_port_fec: None,\n bgp_peers: [],\n autoneg: false,\n },\n ],\n bgp: [],\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.136489743Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:800"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.136497003Z","hostname":"buskin","pid":1661,"component":"DnsResolver","component":"RSS","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.136559515Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"No ledger in /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.136612186Z","hostname":"buskin","pid":1661,"component":"RSS","file":"common/src/ledger.rs:187"} {"msg":"No ledger in /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.136631507Z","hostname":"buskin","pid":1661,"component":"RSS","file":"common/src/ledger.rs:187"} {"msg":"RSS configuration has not been fully applied yet","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.136637757Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:860"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.136657158Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"No ledger in /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.136670258Z","hostname":"buskin","pid":1661,"component":"RSS","file":"common/src/ledger.rs:187"} {"msg":"No ledger in /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.136678828Z","hostname":"buskin","pid":1661,"component":"RSS","file":"common/src/ledger.rs:187"} {"msg":"Creating new allocation plan","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.136692509Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:901"} {"msg":"Serializing plan","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.136695769Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:118"} {"msg":"Creating plan for the sled at fdb0:18c0:4d0d:9fb2::1","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.136698889Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:91"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.13672484Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.136748491Z","hostname":"buskin","pid":1661,"component":"RSS","file":"common/src/ledger.rs:198"} {"msg":"Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.13707051Z","hostname":"buskin","pid":1661,"component":"RSS","file":"common/src/ledger.rs:198"} {"msg":"Sled plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.137238675Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:138"} {"msg":"Writing Rack Network Configuration to bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.137247426Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:934"} {"msg":"Attempting to update network config with generation: 1, current_generation: 0","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.137282096Z","hostname":"buskin","pid":1661,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.137294817Z","hostname":"buskin","pid":1661,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:198"} {"msg":"Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.137445681Z","hostname":"buskin","pid":1661,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:198"} {"msg":"Broadcasting network config with generation 1","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.137564515Z","hostname":"buskin","pid":1661,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"Received initialization request from RSS","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.137597056Z","hostname":"buskin","pid":1661,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:217","target_sled":"[fdb0:18c0:4d0d:9fb2::1]:12346","request":"StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: cac4b398-c394-4c89-a5de-5577a51ce104, rack_id: 90130517-c703-4437-bd46-ac71a8336624, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net(Ipv6Network { addr: fd00:1122:3344:101::, prefix: 64 }) } } }"} {"msg":"Accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.1377164Z","hostname":"buskin","pid":1661,"remote_addr":"[fdb0:18c0:4d0d:9fb2::1]:65261","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:66"} {"msg":"Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: cac4b398-c394-4c89-a5de-5577a51ce104, rack_id: 90130517-c703-4437-bd46-ac71a8336624, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net(Ipv6Network { addr: fd00:1122:3344:101::, prefix: 64 }) } } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.137846133Z","hostname":"buskin","pid":1661,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:364"} {"msg":"KeyManager: using hardcoded secret retriever","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.137851724Z","hostname":"buskin","pid":1661,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:379"} {"msg":"setting up sled agent server","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.137864484Z","hostname":"buskin","pid":1661,"file":"sled-agent/src/server.rs:47"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.137872474Z","hostname":"buskin","pid":1661,"component":"DnsResolver","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"Received KeyManagerReady","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.137876705Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Attempting to add queued disks","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.137880015Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:394","num_disks":"9"} {"msg":"Enabling switch zone (new)","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.14004801Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:3722"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.146585697Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/dataset.rs:327","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.146644619Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/dataset.rs:329","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.14665229Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/dataset.rs:333","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.14667204Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/dataset.rs:335","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.146761173Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:37"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.146770933Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/dataset.rs:347"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.182465431Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:56"} {"msg":"Automatically destroying dataset: oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.190197475Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/dataset.rs:389"} {"msg":"SledAgent::new(..) starting","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.196425423Z","hostname":"buskin","pid":1661,"sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:350"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.196466124Z","hostname":"buskin","pid":1661,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:91","prefix":"Ipv6Prefix { addr: fd00:1122:3344:101::, len: 64 }"} {"msg":"Looking for unencrypted datasets in oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.293839605Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/dataset.rs:474"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.316959913Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/dataset.rs:327","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.316996644Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/dataset.rs:329","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.317008335Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/dataset.rs:333","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.317020795Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/dataset.rs:335","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.317142949Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:37"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.317149019Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/dataset.rs:347"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.344346141Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:56"} {"msg":"Automatically destroying dataset: oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.35162244Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/dataset.rs:389"} {"msg":"Looking for unencrypted datasets in oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.454483267Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/dataset.rs:474"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.476908844Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/dataset.rs:327","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\",\n model: \"synthetic-model\",\n}"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.476982867Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/dataset.rs:329","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.476998877Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/dataset.rs:333","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.477013047Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/dataset.rs:335","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.477156622Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:37"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.477164022Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/dataset.rs:347"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.507961472Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:56"} {"msg":"Automatically destroying dataset: oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.515355185Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/dataset.rs:389"} {"msg":"Looking for unencrypted datasets in oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.617840421Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/dataset.rs:474"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.640481425Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/dataset.rs:327","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.640556557Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/dataset.rs:329","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.640585748Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/dataset.rs:333","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.640596438Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/dataset.rs:335","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.640771793Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:37"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.640783154Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/dataset.rs:347"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.667759448Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:56"} {"msg":"Automatically destroying dataset: oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.675178262Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/dataset.rs:389"} {"msg":"Looking for unencrypted datasets in oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.776443031Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/dataset.rs:474"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.798767045Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/dataset.rs:327","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.798791226Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/dataset.rs:329","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.798797336Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/dataset.rs:333","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.798810327Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/dataset.rs:335","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.79892175Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:37"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.79893081Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/dataset.rs:347"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.826582675Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:56"} {"msg":"Automatically destroying dataset: oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.8350236Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/dataset.rs:389"} {"msg":"Looking for unencrypted datasets in oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.940183296Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/dataset.rs:474"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.962951274Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/dataset.rs:327","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\",\n model: \"synthetic-model\",\n}"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.962991905Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/dataset.rs:329","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.963001095Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/dataset.rs:333","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.963015166Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/dataset.rs:335","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.963179111Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:37"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.963186241Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/dataset.rs:347"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.991099854Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:56"} {"msg":"Automatically destroying dataset: oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:39.998766295Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/dataset.rs:389"} {"msg":"Looking for unencrypted datasets in oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:40.102351174Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/dataset.rs:474"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:40.125007908Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/dataset.rs:327","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\",\n model: \"synthetic-model\",\n}"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:40.125032239Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/dataset.rs:329","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:40.125037619Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/dataset.rs:333","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:40.125111161Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/dataset.rs:335","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:40.125191253Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:37"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:40.125198904Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/dataset.rs:347"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:40.152299582Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:56"} {"msg":"Automatically destroying dataset: oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:40.160047456Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/dataset.rs:389"} {"msg":"Looking for unencrypted datasets in oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:40.266985586Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/dataset.rs:474"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:40.289659451Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/dataset.rs:327","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\",\n model: \"synthetic-model\",\n}"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:40.289727193Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/dataset.rs:329","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:40.289735703Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/dataset.rs:333","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:40.289742653Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/dataset.rs:335","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:40.289878387Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:37"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:40.289884938Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/dataset.rs:347"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:40.317504012Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:56"} {"msg":"Automatically destroying dataset: oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:40.324956247Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/dataset.rs:389"} {"msg":"Looking for unencrypted datasets in oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:40.427444602Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/dataset.rs:474"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:40.45053951Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/dataset.rs:327","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\",\n model: \"synthetic-model\",\n}"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:40.450574391Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/dataset.rs:329","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:40.450581921Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/dataset.rs:333","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:40.450614252Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/dataset.rs:335","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:40.450756396Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:37"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:40.450764137Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/dataset.rs:347"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:40.479123793Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:56"} {"msg":"Automatically destroying dataset: oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:40.48663455Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/dataset.rs:389"} {"msg":"Looking for unencrypted datasets in oxp_31bd71cd-4736-4a12-a387-9b74b050396f","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:40.590086194Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/dataset.rs:474"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:40.605121799Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:40.605141739Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Requested swap device of size 64 GiB","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:40.60516714Z","hostname":"buskin","pid":1661,"sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:362"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:40.610650855Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:111","resources":"StorageResources { disks: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }), Pool { name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, info: ZpoolInfo { name: \"oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03\", size: 20937965568, allocated: 817152, free: 20937148416, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }), Pool { name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, info: ZpoolInfo { name: \"oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03\", size: 20937965568, allocated: 772608, free: 20937192960, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }), Pool { name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, info: ZpoolInfo { name: \"oxp_31bd71cd-4736-4a12-a387-9b74b050396f\", size: 20937965568, allocated: 777216, free: 20937188352, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } }), Pool { name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, info: ZpoolInfo { name: \"oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7\", size: 20937965568, allocated: 774144, free: 20937191424, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: a462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal } }), Pool { name: ZpoolName { id: a462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal }, info: ZpoolInfo { name: \"oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b\", size: 20937965568, allocated: 508416, free: 20937457152, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: b462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal } }), Pool { name: ZpoolName { id: b462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal }, info: ZpoolInfo { name: \"oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b\", size: 20937965568, allocated: 500736, free: 20937464832, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } }), Pool { name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, info: ZpoolInfo { name: \"oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", size: 20937965568, allocated: 1208832, free: 20936756736, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } }), Pool { name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, info: ZpoolInfo { name: \"oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60\", size: 20937965568, allocated: 823296, free: 20937142272, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } }), Pool { name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, info: ZpoolInfo { name: \"oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b\", size: 20937965568, allocated: 1219584, free: 20936745984, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }), Pool { name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, info: ZpoolInfo { name: \"oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", size: 20937965568, allocated: 772608, free: 20937192960, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }), Pool { name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, info: ZpoolInfo { name: \"oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", size: 20937965568, allocated: 786432, free: 20937179136, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" } })} }"} {"msg":"attempting to create encrypted zvol: name=\"oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:40.627986219Z","hostname":"buskin","pid":1661,"file":"sled-agent/src/swap_device.rs:153"} {"msg":"Configuring new Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:40.63001139Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:40.648265351Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"successfully created encrypted zvol: name=\"oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:40.650219841Z","hostname":"buskin","pid":1661,"file":"sled-agent/src/swap_device.rs:249"} {"msg":"adding swap device: swapname=\"/dev/zvol/dsk/oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b/swap\"","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:40.650354035Z","hostname":"buskin","pid":1661,"file":"sled-agent/src/swap_device.rs:96"} {"msg":"Mounting backing filesystems","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:40.655254283Z","hostname":"buskin","pid":1661,"sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:377"} {"msg":"Processing fmd","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:40.655261553Z","hostname":"buskin","pid":1661,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:126"} {"msg":"Ensuring dataset oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b/backing/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:40.655268393Z","hostname":"buskin","pid":1661,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:136"} {"msg":"Stopping service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:40.716903805Z","hostname":"buskin","pid":1661,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:170"} {"msg":"Mounting oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b/backing/fmd on /var/fm/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:40.935109375Z","hostname":"buskin","pid":1661,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:178"} {"msg":"Ensuring directory /var/fm/fmd/rsrc","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:40.947555641Z","hostname":"buskin","pid":1661,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Ensuring directory /var/fm/fmd/ckpt","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:40.947695155Z","hostname":"buskin","pid":1661,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Ensuring directory /var/fm/fmd/xprt","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:40.947737947Z","hostname":"buskin","pid":1661,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Starting service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:40.947754587Z","hostname":"buskin","pid":1661,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:192"} {"msg":"Ensuring contract reaper thread","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:41.130266979Z","hostname":"buskin","pid":1661,"file":"illumos-utils/src/running_zone.rs:163"} {"msg":"using '[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]' as data links for xde driver","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:41.675046193Z","hostname":"buskin","pid":1661,"sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"illumos-utils/src/opte/illumos.rs:90"} {"msg":"Underlay Available","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:41.734921381Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:122","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104"} {"msg":"50% of 102952026112 physical ram = 51476013056 bytes)","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:41.734973472Z","hostname":"buskin","pid":1661,"component":"VmmReservoirManager","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:252"} {"msg":"Setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:41.735001113Z","hostname":"buskin","pid":1661,"component":"VmmReservoirManager","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:260"} {"msg":"sled agent started","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:41.735076546Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:979","underlay_address":"fd00:1122:3344:101::1"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:41.735083306Z","hostname":"buskin","pid":1661,"component":"DnsResolver","component":"ServiceManager","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:41.736059035Z","hostname":"buskin","pid":1661,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:195"} {"msg":"Sled Agent loaded; recording configuration","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:41.736092716Z","hostname":"buskin","pid":1661,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:424"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:41.736135898Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:41.736158728Z","hostname":"buskin","pid":1661,"component":"SledAgentMain","file":"common/src/ledger.rs:198"} {"msg":"Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:41.736430356Z","hostname":"buskin","pid":1661,"component":"SledAgentMain","file":"common/src/ledger.rs:198"} {"msg":"Sled Agent Started","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:41.736618192Z","hostname":"buskin","pid":1661,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:137"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:41.736635013Z","hostname":"buskin","pid":1661,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:243","underlay_network_info":"Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV1 { rack_subnet: Ipv6Network { addr: fd00:1122:3344:100::, prefix: 56 }, infra_ip_first: 192.168.1.30, infra_ip_last: 192.168.1.30, ports: [PortConfigV1 { routes: [RouteConfig { destination: V4(Ipv4Network { addr: 0.0.0.0, prefix: 0 }), nexthop: 10.151.1.100 }], addresses: [V4(Ipv4Network { addr: 10.151.1.174, prefix: 24 })], switch: Switch0, port: \"qsfp0\", uplink_port_speed: Speed40G, uplink_port_fec: None, bgp_peers: [], autoneg: false }], bgp: [] })))"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:41.736640963Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:3494"} {"msg":"Connection closed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:41.736720475Z","hostname":"buskin","pid":1661,"remote_addr":"[fdb0:18c0:4d0d:9fb2::1]:65261","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:73"} {"msg":"Peer agent initialized","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:41.921324141Z","hostname":"buskin","pid":1661,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:108","peer":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Initialized sled agent","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:41.921391643Z","hostname":"buskin","pid":1661,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:232","target_sled":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:41.921688732Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"No ledger in /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/rss-service-plan-v2.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:41.921773454Z","hostname":"buskin","pid":1661,"component":"RSS","file":"common/src/ledger.rs:187"} {"msg":"No ledger in /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/rss-service-plan-v2.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:41.921794035Z","hostname":"buskin","pid":1661,"component":"RSS","file":"common/src/ledger.rs:187"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:41.921833156Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:41.923528797Z","hostname":"buskin","pid":1661,"sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1120","retry_after":"247.487809ms","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:41.926128226Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:41.927061734Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:41.928148927Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:41.92989836Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:41.930962702Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:41.932134377Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:41.93290395Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: cac4b398-c394-4c89-a5de-5577a51ce104, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:41.933473638Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:41.933665324Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104/zpools/14b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:41.933964262Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"failed to notify nexus about a new pool oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:41.933992523Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104/zpools/a462a7f7-b628-40fe-80ff-4e4189e2d62b): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:41.934284162Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104/zpools/e4b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:41.934413336Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104/zpools/f4b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", sled_id: cac4b398-c394-4c89-a5de-5577a51ce104, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:41.934658703Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", sled_id: cac4b398-c394-4c89-a5de-5577a51ce104, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:41.934793318Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: cac4b398-c394-4c89-a5de-5577a51ce104, variant: M2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:41.934980163Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:41.934988503Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", sled_id: cac4b398-c394-4c89-a5de-5577a51ce104, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:41.935105437Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", sled_id: cac4b398-c394-4c89-a5de-5577a51ce104, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:41.935232761Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: cac4b398-c394-4c89-a5de-5577a51ce104, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:41.935346424Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: cac4b398-c394-4c89-a5de-5577a51ce104, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:41.935457807Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:41.935589691Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104/zpools/24b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxp_31bd71cd-4736-4a12-a387-9b74b050396f on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:41.935723656Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104/zpools/31bd71cd-4736-4a12-a387-9b74b050396f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:41.93585841Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104/zpools/616b26df-e62a-4c68-b506-f4a923d8aaf7): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:41.936012564Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104/zpools/b462a7f7-b628-40fe-80ff-4e4189e2d62b): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:41.936118067Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"failed to notify nexus about a new pool oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:41.936154339Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104/zpools/cd70d7f6-2354-4bf2-8012-55bf9eaf7930): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:41.936268502Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104/zpools/ceb4461c-cf56-4719-ad3c-14430bfdfb60): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:41.936408266Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104/zpools/d462a7f7-b628-40fe-80ff-4e4189e2d62b): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: cac4b398-c394-4c89-a5de-5577a51ce104, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:41.936553861Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: cac4b398-c394-4c89-a5de-5577a51ce104, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:41.936676234Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: cac4b398-c394-4c89-a5de-5577a51ce104, variant: M2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:41.936818769Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:41.93686387Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:41.938358085Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:41.940870171Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:41.942185731Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:41.943562952Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:41.945033627Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:41.946447079Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:41.947827441Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:41.949352687Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:41.951838492Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:41.953198073Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:41.954675188Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:41.955436551Z","hostname":"buskin","pid":1661,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:59316"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:41.955608186Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:41.955657018Z","hostname":"buskin","pid":1661,"uri":"/zpools","method":"GET","req_id":"ffa2a61b-a507-4c02-bf84-4832d234c0fd","remote_addr":"[fd00:1122:3344:101::1]:59316","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":110,"response_code":"200"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:41.955972047Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:41.957305898Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:41.959762732Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:41.961129983Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:41.962548456Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:41.963985029Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:41.965387011Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:41.972884988Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:41.974257619Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:41.97561894Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:41.977033083Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:41.978452586Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:41.979868828Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:41.981220139Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:41.982663183Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:41.984088356Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:41.985499069Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:41.986014264Z","hostname":"buskin","pid":1661,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:39681"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:41.986127897Z","hostname":"buskin","pid":1661,"uri":"/sled-role","method":"GET","req_id":"b614d083-eb72-4189-9e9e-8fcd64cc83f1","remote_addr":"[fd00:1122:3344:101::1]:39681","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":60,"response_code":"200"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:41.987943172Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:41.989444258Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:41.989539531Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.rss-service-plan-v2.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:41.989565901Z","hostname":"buskin","pid":1661,"component":"RSS","file":"common/src/ledger.rs:198"} {"msg":"Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.rss-service-plan-v2.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:41.989912162Z","hostname":"buskin","pid":1661,"component":"RSS","file":"common/src/ledger.rs:198"} {"msg":"Service plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:41.990105198Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/plan/service.rs:741"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:41.991246602Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:41.992627114Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:41.994027876Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:41.995445229Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:41.996849031Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:41.998290635Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:41.999725878Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:42.001286975Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:42.002648747Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:42.004162502Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:42.005590145Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:42.007136022Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:42.00873362Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:42.010472763Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:42.012200985Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:42.013655549Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:42.015284458Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:42.016649139Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:42.018090363Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:42.019491825Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:42.020846716Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:42.022249528Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: 6507a4bd-9097-45d0-9787-1240feff9525, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:42.022510816Z","hostname":"buskin","pid":1661,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:42.022712522Z","hostname":"buskin","pid":1661,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:58446"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6507a4bd-9097-45d0-9787-1240feff9525, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:42.02296395Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6507a4bd-9097-45d0-9787-1240feff9525, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:42.022987851Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:42.114969049Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:42.116648839Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:42.118059812Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:42.119201027Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:42.120081113Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:42.305695709Z","hostname":"buskin","pid":1661,"sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1120","retry_after":"489.823612ms","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:42.30572484Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:42.307241586Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:42.308262577Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:42.309181974Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:42.309942597Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:42.311270648Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:42.312796154Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:42.313794844Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:42.315270669Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:42.315948789Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:42.31697779Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:42.31865004Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:42.320184907Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:42.321585999Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:42.323088105Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:42.324558689Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:42.501246215Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:42.503489223Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:42.505007909Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:42.505964698Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:42.507052481Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:42.508117013Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:42.704169114Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:42.705427552Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:42.707199596Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:42.709964929Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:42.711150325Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:42.711885857Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:42.713776764Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:42.715082554Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:42.715091744Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:42.93744412Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:42.9384367Z","hostname":"buskin","pid":1661,"sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1120","retry_after":"765.752713ms","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:42.940456871Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:42.942860483Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:42.944692889Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:43.146664718Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:43.148257746Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:43.149226425Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:43.150164824Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:43.151111352Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:43.151802143Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:43.153152344Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:43.153935947Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:43.154836145Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:43.156098453Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:43.157199176Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:43.158517106Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:43.160074083Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:43.161528177Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:43.163014882Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:43.164428495Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:43.332737248Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:43.334195942Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:43.335294605Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:43.336034747Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:43.337170752Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:43.338879173Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:43.534251974Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:43.535755499Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:43.537197703Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:43.537926515Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:43.538949776Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:43.539911455Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:43.54106874Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:43.54206191Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:43.543125652Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:43.543973818Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:43.544820563Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:43.545447662Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:43.545462853Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:43.729491951Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:43.730820431Z","hostname":"buskin","pid":1661,"sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1120","retry_after":"1.967474469s","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:43.731309866Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:43.733050518Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:43.733925565Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:43.928368537Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:43.930199743Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:43.931257805Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:43.931905924Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:43.933285736Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:43.934507283Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:43.936268946Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:43.93771841Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:43.939459523Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:43.940864385Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:43.942235216Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:43.943652279Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:43.945110333Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:43.946556907Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:43.947927408Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:43.949365562Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:43.951786995Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:43.953168197Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:44.183943806Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:44.186323448Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:44.188958998Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:44.190479673Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:44.191939368Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:44.192876076Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:44.193950668Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:44.194891277Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:44.195831785Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:44.196915278Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:44.430823112Z","hostname":"buskin","pid":1661,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:44.627826502Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:44.629198374Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"No ledger in /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:44.629255746Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:187"} {"msg":"No ledger in /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:44.629307237Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:187"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:44.629364089Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 29.840978ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:45.198492017Z","hostname":"buskin","pid":1661,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 60.351394ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:45.742206339Z","hostname":"buskin","pid":1661,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:45.743849428Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:45.745469027Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:45.745572011Z","hostname":"buskin","pid":1661,"sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1120","retry_after":"2.364043048s","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:45.747384505Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:45.749089287Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:45.749953893Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:45.751309284Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:45.753583193Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:45.754872992Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 294.010191ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:46.376159996Z","hostname":"buskin","pid":1661,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:46.378669611Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:46.381030543Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:46.382069434Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:46.383450096Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:46.385435356Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:46.386948651Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:46.388641662Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:46.389532549Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:46.38954621Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:46.38955034Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:46.564730831Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:46.5663595Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:47.148588134Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 558.735377ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:47.148685307Z","hostname":"buskin","pid":1661,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:47.379965793Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:47.381767917Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:47.383007434Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:47.38385934Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:47.384867341Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:47.595458031Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:47.597675058Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:47.598921886Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_6e31b68c-68c6-4db7-adbb-2ed9a659e17e","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:48.05205202Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 445.498219ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:48.326029965Z","hostname":"buskin","pid":1661,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:48.326119138Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:48.550844865Z","hostname":"buskin","pid":1661,"sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1120","retry_after":"6.012424291s","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:48.551353691Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:48.553015421Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:48.554003821Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:49.264806378Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.389094978s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:49.265139288Z","hostname":"buskin","pid":1661,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_6507a4bd-9097-45d0-9787-1240feff9525","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:49.265322314Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:49.50788116Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Installing Omicron zone: oxz_internal_dns_6e31b68c-68c6-4db7-adbb-2ed9a659e17e","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:49.508027084Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:49.766011626Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:49.766058457Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:50.036744792Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Installing Omicron zone: oxz_internal_dns_6507a4bd-9097-45d0-9787-1240feff9525","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:50.036895697Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:50.296938451Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:50.591037093Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:50.593528589Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:50.595147307Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:50.596018674Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:50.597433676Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:50.598829019Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:50.599981044Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:50.601621613Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:50.603046326Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:50.604477929Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:50.605845921Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:50.607348476Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:50.609744508Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:50.611173302Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:50.612586994Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:50.613956635Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:50.615368618Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:50.617800102Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:50.619224605Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:50.620650198Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:50.62206666Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:50.623544445Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:50.624975188Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:50.626415732Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:50.627817534Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:50.629249508Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:50.63066902Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:50.631999251Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:50.633416553Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:50.634848886Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:50.636173717Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:50.6376053Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:50.638994722Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:50.640497897Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:50.6419084Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:50.643325493Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:50.645783727Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:50.647173499Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:50.648279052Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:50.649684005Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:50.651068607Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:50.652466339Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:50.653792119Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:50.879765934Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:50.881262749Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:50.882682692Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:50.884188517Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:51.178232817Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:51.179186186Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:51.180574398Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:51.182001611Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:51.183115655Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:51.184175987Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:51.185064544Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:51.185949721Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Finished setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:52.589401488Z","hostname":"buskin","pid":1661,"component":"VmmReservoirManager","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:264"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:52.591037327Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:52.592379148Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Profile for oxz_internal_dns_6e31b68c-68c6-4db7-adbb-2ed9a659e17e:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:53.052070101Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:53.528050947Z","hostname":"buskin","pid":1661,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:91","prefix":"Ipv6Prefix { addr: fd00:1122:3344:3::, len: 64 }"} {"msg":"Profile for oxz_internal_dns_03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:53.528117599Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:53.52817237Z","hostname":"buskin","pid":1661,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:91","prefix":"Ipv6Prefix { addr: fd00:1122:3344:2::, len: 64 }"} {"msg":"Profile for oxz_internal_dns_6507a4bd-9097-45d0-9787-1240feff9525:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:54.007207178Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:54.007338542Z","hostname":"buskin","pid":1661,"zone":"oxz_internal_dns_6e31b68c-68c6-4db7-adbb-2ed9a659e17e","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:54.011393534Z","hostname":"buskin","pid":1661,"zone":"oxz_internal_dns_03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:54.015581091Z","hostname":"buskin","pid":1661,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:91","prefix":"Ipv6Prefix { addr: fd00:1122:3344:1::, len: 64 }"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:54.015621942Z","hostname":"buskin","pid":1661,"zone":"oxz_internal_dns_6507a4bd-9097-45d0-9787-1240feff9525","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Ensuring tfportrear0_0/ll exists in zone","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:54.033929475Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2277"} {"msg":"wait for service Some(\"oxz_internal_dns_6e31b68c-68c6-4db7-adbb-2ed9a659e17e\") failed: Property not found. retry in 30.064433ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:54.15196778Z","hostname":"buskin","pid":1661,"zone":"oxz_internal_dns_6e31b68c-68c6-4db7-adbb-2ed9a659e17e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_6507a4bd-9097-45d0-9787-1240feff9525\") failed: Property not found. retry in 39.152413ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:54.158792106Z","hostname":"buskin","pid":1661,"zone":"oxz_internal_dns_6507a4bd-9097-45d0-9787-1240feff9525","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3\") failed: Property not found. retry in 65.130855ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:54.165180889Z","hostname":"buskin","pid":1661,"zone":"oxz_internal_dns_03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_6e31b68c-68c6-4db7-adbb-2ed9a659e17e\") failed: Property not found. retry in 77.43903ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:54.190054011Z","hostname":"buskin","pid":1661,"zone":"oxz_internal_dns_6e31b68c-68c6-4db7-adbb-2ed9a659e17e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_6507a4bd-9097-45d0-9787-1240feff9525\") failed: Property not found. retry in 59.492968ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:54.205517358Z","hostname":"buskin","pid":1661,"zone":"oxz_internal_dns_6507a4bd-9097-45d0-9787-1240feff9525","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_6507a4bd-9097-45d0-9787-1240feff9525\") failed: Property not found. retry in 112.822017ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:54.366425007Z","hostname":"buskin","pid":1661,"zone":"oxz_internal_dns_6507a4bd-9097-45d0-9787-1240feff9525","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_6e31b68c-68c6-4db7-adbb-2ed9a659e17e\") failed: Property not found. retry in 281.10783ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:54.391491684Z","hostname":"buskin","pid":1661,"zone":"oxz_internal_dns_6e31b68c-68c6-4db7-adbb-2ed9a659e17e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3\") failed: Property not found. retry in 83.021534ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:54.399107034Z","hostname":"buskin","pid":1661,"zone":"oxz_internal_dns_03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_6507a4bd-9097-45d0-9787-1240feff9525\") failed: Property not found. retry in 248.642171ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:54.511664384Z","hostname":"buskin","pid":1661,"zone":"oxz_internal_dns_6507a4bd-9097-45d0-9787-1240feff9525","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3\") failed: Property not found. retry in 194.651056ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:54.534542285Z","hostname":"buskin","pid":1661,"zone":"oxz_internal_dns_03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_6e31b68c-68c6-4db7-adbb-2ed9a659e17e\") failed: Property not found. retry in 398.917014ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:54.680196294Z","hostname":"buskin","pid":1661,"zone":"oxz_internal_dns_6e31b68c-68c6-4db7-adbb-2ed9a659e17e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3\") failed: Property not found. retry in 265.185316ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:54.746011782Z","hostname":"buskin","pid":1661,"zone":"oxz_internal_dns_03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_6507a4bd-9097-45d0-9787-1240feff9525\") failed: Property not found. retry in 1.18244574s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:54.782104522Z","hostname":"buskin","pid":1661,"zone":"oxz_internal_dns_6507a4bd-9097-45d0-9787-1240feff9525","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3\") failed: Property not found. retry in 1.006448806s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:55.0221108Z","hostname":"buskin","pid":1661,"zone":"oxz_internal_dns_03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_6e31b68c-68c6-4db7-adbb-2ed9a659e17e\") failed: Property not found. retry in 992.707002ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:55.090068973Z","hostname":"buskin","pid":1661,"zone":"oxz_internal_dns_6e31b68c-68c6-4db7-adbb-2ed9a659e17e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_6507a4bd-9097-45d0-9787-1240feff9525\") failed: Property not found. retry in 1.21269399s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:55.975246697Z","hostname":"buskin","pid":1661,"zone":"oxz_internal_dns_6507a4bd-9097-45d0-9787-1240feff9525","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3\") failed: Property not found. retry in 1.363281891s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:56.040297631Z","hostname":"buskin","pid":1661,"zone":"oxz_internal_dns_03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_6e31b68c-68c6-4db7-adbb-2ed9a659e17e\") failed: Property not found. retry in 1.407383537s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:56.094193089Z","hostname":"buskin","pid":1661,"zone":"oxz_internal_dns_6e31b68c-68c6-4db7-adbb-2ed9a659e17e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_6507a4bd-9097-45d0-9787-1240feff9525\") failed: Property not found. retry in 647.483681ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:57.197193652Z","hostname":"buskin","pid":1661,"zone":"oxz_internal_dns_6507a4bd-9097-45d0-9787-1240feff9525","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3\") failed: Property not found. retry in 724.26807ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:57.412143724Z","hostname":"buskin","pid":1661,"zone":"oxz_internal_dns_03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_6e31b68c-68c6-4db7-adbb-2ed9a659e17e\") failed: Property not found. retry in 1.029358725s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:57.510353231Z","hostname":"buskin","pid":1661,"zone":"oxz_internal_dns_6e31b68c-68c6-4db7-adbb-2ed9a659e17e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:57.594753589Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"wait for service Some(\"oxz_internal_dns_6507a4bd-9097-45d0-9787-1240feff9525\") failed: Property not found. retry in 647.877423ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:57.854180175Z","hostname":"buskin","pid":1661,"zone":"oxz_internal_dns_6507a4bd-9097-45d0-9787-1240feff9525","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3\") failed: Property not found. retry in 1.000590863s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:58.146906836Z","hostname":"buskin","pid":1661,"zone":"oxz_internal_dns_03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_6507a4bd-9097-45d0-9787-1240feff9525\") failed: Property not found. retry in 1.145003408s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:58.510882429Z","hostname":"buskin","pid":1661,"zone":"oxz_internal_dns_6507a4bd-9097-45d0-9787-1240feff9525","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_6e31b68c-68c6-4db7-adbb-2ed9a659e17e\") failed: Property not found. retry in 1.376984465s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:43:58.549596378Z","hostname":"buskin","pid":1661,"zone":"oxz_internal_dns_6e31b68c-68c6-4db7-adbb-2ed9a659e17e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:59.947070345Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2908","zone":"oxz_internal_dns_6507a4bd-9097-45d0-9787-1240feff9525"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:59.947117396Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2908","zone":"oxz_internal_dns_6e31b68c-68c6-4db7-adbb-2ed9a659e17e"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:59.947123716Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2908","zone":"oxz_internal_dns_03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3"} {"msg":"Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:59.947169608Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:198"} {"msg":"Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:59.947395954Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:198"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:59.947667732Z","hostname":"buskin","pid":1661,"uri":"/omicron-zones","method":"PUT","req_id":"ac6f26d8-a15d-46be-8c12-bbe95e9bd98b","remote_addr":"[fd00:1122:3344:101::1]:58446","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":17924861,"response_code":"204"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:43:59.949408445Z","hostname":"buskin","pid":1661,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:417"} {"msg":"failed to write DNS configuration (will retry in 118.04569ms)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:00.018242423Z","hostname":"buskin","pid":1661,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:437","error_message":"Communication Error: error sending request for url (http://[fd00:1122:3344:1::1]:5353/config): error trying to connect: tcp connect error: Connection refused (os error 146)"} {"msg":"Ensuring bootstrap address fdb0:18c0:4d0d:9fb2::2 exists in switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:00.073452321Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2294"} {"msg":"Adding bootstrap address","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:00.073470631Z","hostname":"buskin","pid":1661,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:626"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:00.156366445Z","hostname":"buskin","pid":1661,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:417"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:00.202514569Z","hostname":"buskin","pid":1661,"dns_config_addr":"[fd00:1122:3344:3::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:417"} {"msg":"Configured all DNS servers","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:00.24692639Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:453"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:00.246938011Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:254"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:00.280246847Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:271","addr":"fd00:1122:3344:101::2"} {"msg":"Forwarding bootstrap traffic via oxBootstrap0 to fe80::8:20ff:feb1:41ce","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:00.624311578Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2301"} {"msg":"Zone not using gateway (even though underlay is up)","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:00.630524766Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2362"} {"msg":"Setting up dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:00.753670905Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2538"} {"msg":"Setting up MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:01.000814099Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2417"} {"msg":"Setting up wicketd service","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:01.290161008Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2452"} {"msg":"Setting up mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:01.483294791Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2719"} {"msg":"Setting up mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:01.630961471Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2724"} {"msg":"Setting up tfport service","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:01.810419211Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2658"} {"msg":"Setting up Simulated SP service","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:01.979459336Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2446"} {"msg":"Re-enabling running switch zone (new address)","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:01.989115998Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:3746","new":"[fd00:1122:3344:101::2, ::1]","old":"[::1]"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:01.989141448Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:3762"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:01.989149969Z","hostname":"buskin","pid":1661,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:02.53517182Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:3770"} {"msg":"configuring dendrite zone","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:02.541166721Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:3826"} {"msg":"configuring wicketd","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:02.721272251Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:3866","rack_subnet":"fd00:1122:3344::"} {"msg":"Determining physical location of our switch zone at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:02.773769456Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:360"} {"msg":"Failed to get switch ID from MGS (retrying in 38.831568ms)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:02.805273278Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:379","error":"Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5a6ea208-0869-4065-b4d9-ca5c81f26f4d\", \"content-length\": \"152\", \"date\": \"Thu, 07 Mar 2024 22:44:02 GMT\"}; value: Error { error_code: Some(\"DiscoveryNotYetComplete\"), message: \"discovery process not yet complete\", request_id: \"5a6ea208-0869-4065-b4d9-ca5c81f26f4d\" }"} {"msg":"Failed to get switch ID from MGS (retrying in 134.302571ms)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:02.844863404Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:379","error":"Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6a35334e-96ac-401b-8ee6-6c402bb4191b\", \"content-length\": \"152\", \"date\": \"Thu, 07 Mar 2024 22:44:02 GMT\"}; value: Error { error_code: Some(\"DiscoveryNotYetComplete\"), message: \"discovery process not yet complete\", request_id: \"6a35334e-96ac-401b-8ee6-6c402bb4191b\" }"} {"msg":"Failed to get switch ID from MGS (retrying in 268.801847ms)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:02.979945363Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:379","error":"Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0b230fd0-077b-476c-a207-ec2b7d3ac12c\", \"content-length\": \"152\", \"date\": \"Thu, 07 Mar 2024 22:44:02 GMT\"}; value: Error { error_code: Some(\"DiscoveryNotYetComplete\"), message: \"discovery process not yet complete\", request_id: \"0b230fd0-077b-476c-a207-ec2b7d3ac12c\" }"} {"msg":"Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:03.249751621Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:408"} {"msg":"Building Port Configuration","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:03.280927803Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:567"} {"msg":"Checking dendrite uptime","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:03.281015496Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:609"} {"msg":"Dendrite online","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:03.281442259Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:612","uptime":"3"} {"msg":"Configuring default uplink on switch","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:03.281469259Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:430","config":"PortSettings {\n links: {\n \"0\": LinkSettings {\n addrs: [\n 10.151.1.174,\n ],\n params: LinkCreate {\n autoneg: false,\n fec: None,\n kr: false,\n lane: Some(\n LinkId(\n 0,\n ),\n ),\n speed: Speed40G,\n },\n },\n },\n v4_routes: {},\n v6_routes: {},\n}"} {"msg":"Received DisksChanged({})","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:03.357152825Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:03.67748411Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:296"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:03.677499291Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:188","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: 6507a4bd-9097-45d0-9787-1240feff9525, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 65847932-72ba-40f7-b62c-5adce2529400, kind: Service { id: f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf }, name: Name(\"ntp-f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:03.706891138Z","hostname":"buskin","pid":1661,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:03.707071994Z","hostname":"buskin","pid":1661,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:50734"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6507a4bd-9097-45d0-9787-1240feff9525, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:03.707259119Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6507a4bd-9097-45d0-9787-1240feff9525, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:03.70727644Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:03.727079568Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:03.727115799Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:03.745133954Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:03.745169564Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:03.763716875Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:03.763756166Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:03.763930651Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:03.764034204Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:03.774784839Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:254"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:03.805751694Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:271","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:03.806416134Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:296"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:03.806421714Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:188","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:03.850069763Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte0\", ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 157, 105]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 }, vnic: \"vopte0\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:03.850119054Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:1269","zone_type":"ntp"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.1.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:03.850740513Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:98"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.1.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:03.850751483Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:105"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.1.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:03.851613859Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:133"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:03.854367422Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Configuring new Omicron zone: oxz_ntp_f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:03.879019567Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_ntp_f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:03.896718922Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_ntp_f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:05.095733314Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:05.095876878Z","hostname":"buskin","pid":1661,"zone":"oxz_ntp_f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_ntp_f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf\") failed: Property not found. retry in 26.946903ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:05.244772095Z","hostname":"buskin","pid":1661,"zone":"oxz_ntp_f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf\") failed: Property not found. retry in 55.704465ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:05.279974278Z","hostname":"buskin","pid":1661,"zone":"oxz_ntp_f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf\") failed: Property not found. retry in 202.093278ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:05.346705134Z","hostname":"buskin","pid":1661,"zone":"oxz_ntp_f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf\") failed: Property not found. retry in 288.43591ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:05.560043557Z","hostname":"buskin","pid":1661,"zone":"oxz_ntp_f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf\") failed: Property not found. retry in 927.40204ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:05.858380378Z","hostname":"buskin","pid":1661,"zone":"oxz_ntp_f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:06.571187326Z","hostname":"buskin","pid":1661,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:52637"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:06.571359591Z","hostname":"buskin","pid":1661,"uri":"/baseboard","method":"GET","req_id":"0f54d385-c925-466a-a135-c8bfd8d708a7","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:52637","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":86,"response_code":"200"} {"msg":"wait for service Some(\"oxz_ntp_f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf\") failed: Property not found. retry in 1.367020555s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:06.797292315Z","hostname":"buskin","pid":1661,"zone":"oxz_ntp_f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf\") failed: Property not found. retry in 549.380228ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:08.173097076Z","hostname":"buskin","pid":1661,"zone":"oxz_ntp_f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf\") failed: Property not found. retry in 970.316728ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:08.732459151Z","hostname":"buskin","pid":1661,"zone":"oxz_ntp_f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf\") failed: Property not found. retry in 1.45774642s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:09.711715516Z","hostname":"buskin","pid":1661,"zone":"oxz_ntp_f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:11.190849099Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2908","zone":"oxz_ntp_f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf"} {"msg":"Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:11.19089935Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:198"} {"msg":"Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:11.191172268Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:198"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:11.191423756Z","hostname":"buskin","pid":1661,"uri":"/omicron-zones","method":"PUT","req_id":"a4bee2ad-1baf-48ae-b0ef-3ac80f6884a8","remote_addr":"[fd00:1122:3344:101::1]:50734","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":7484281,"response_code":"204"} {"msg":"Waiting for rack time synchronization","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:11.19354958Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:494"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:11.222184985Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:11.222300879Z","hostname":"buskin","pid":1661,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:42099"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:11.230338311Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:3480"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:11.230474505Z","hostname":"buskin","pid":1661,"uri":"/timesync","method":"GET","req_id":"132ccf9c-0304-4b24-b684-e4fe658803c9","remote_addr":"[fd00:1122:3344:101::1]:42099","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8063,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:11.232341872Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:11.403869082Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:11.403963075Z","hostname":"buskin","pid":1661,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:53563"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:15.412250653Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:3480"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:15.412355687Z","hostname":"buskin","pid":1661,"uri":"/timesync","method":"GET","req_id":"f8aed5bf-9eb5-4745-a5b9-0ab66816046b","remote_addr":"[fd00:1122:3344:101::1]:53563","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4008344,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:15.415292155Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:15.606654455Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:15.606776939Z","hostname":"buskin","pid":1661,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:44646"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:15.614920545Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:3480"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:15.615019058Z","hostname":"buskin","pid":1661,"uri":"/timesync","method":"GET","req_id":"ca0d79f3-93d8-48c5-a361-912bf790b246","remote_addr":"[fd00:1122:3344:101::1]:44646","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8165,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:15.616771801Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:15.744009084Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:15.744101596Z","hostname":"buskin","pid":1661,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:54367"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:19.752465287Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:3480"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:19.752648422Z","hostname":"buskin","pid":1661,"uri":"/timesync","method":"GET","req_id":"1ae70254-7460-443a-9b5f-dd68312f70a0","remote_addr":"[fd00:1122:3344:101::1]:54367","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4008497,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:19.754834639Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:20.040234188Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:20.040358351Z","hostname":"buskin","pid":1661,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:45552"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:20.049764346Z","hostname":"buskin","pid":1661,"uri":"/timesync","method":"GET","req_id":"bd45d6d7-762a-4e8e-893f-e0b0b2d513b9","remote_addr":"[fd00:1122:3344:101::1]:45552","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":9301,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1709851455.1148555, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:20.051654292Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:20.051661333Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:20.313361407Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:20.313506471Z","hostname":"buskin","pid":1661,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:33872"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:20.321761521Z","hostname":"buskin","pid":1661,"uri":"/timesync","method":"GET","req_id":"bba15216-f747-4949-9871-69762d59ef55","remote_addr":"[fd00:1122:3344:101::1]:33872","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":8200,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1709851455.1148555, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:20.323579396Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:20.324111701Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:20.60385833Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:20.603956193Z","hostname":"buskin","pid":1661,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:64517"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:20.611296575Z","hostname":"buskin","pid":1661,"uri":"/timesync","method":"GET","req_id":"40613f19-2cb3-47d1-a14d-6c61824450f5","remote_addr":"[fd00:1122:3344:101::1]:64517","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":7295,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1709851455.1148555, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:20.61278404Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:20.61278835Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:20.875040091Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:20.875124383Z","hostname":"buskin","pid":1661,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:35926"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:20.882906129Z","hostname":"buskin","pid":1661,"uri":"/timesync","method":"GET","req_id":"4b0a2b4f-07d9-4ef3-bc01-9d558e3f6b24","remote_addr":"[fd00:1122:3344:101::1]:35926","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":7727,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1709851455.1148555, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:20.884756434Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:20.884760845Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:21.373544907Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:21.373638389Z","hostname":"buskin","pid":1661,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:51879"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:21.381325462Z","hostname":"buskin","pid":1661,"uri":"/timesync","method":"GET","req_id":"d544277c-0847-4ed9-8846-47598d5d5842","remote_addr":"[fd00:1122:3344:101::1]:51879","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":7641,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1709851455.1148555, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:21.382894319Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:21.382898529Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:22.018884531Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:22.018988034Z","hostname":"buskin","pid":1661,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:38531"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:22.022291864Z","hostname":"buskin","pid":1661,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:776"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:22.039858644Z","hostname":"buskin","pid":1661,"uri":"/timesync","method":"GET","req_id":"982322b0-e210-4d26-8394-059ee679b996","remote_addr":"[fd00:1122:3344:101::1]:38531","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":20755,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: true, ref_id: 2728380417, ip_addr: 162.159.200.1, stratum: 4, ref_time: 1709851461.6851516, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:22.041592147Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Finished setting up Internal DNS and NTP","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:22.041597537Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1039"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 6507a4bd-9097-45d0-9787-1240feff9525, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 65847932-72ba-40f7-b62c-5adce2529400, kind: Service { id: f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf }, name: Name(\"ntp-f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 416eb39a-1cea-4752-bc22-195679c44853, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bdd869e5-e908-42b7-afb7-35517ea5ca46, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bd70b00d-6277-40dc-9a7a-524663f1a340, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 777ca252-0e2b-4952-8163-fd713097cfd8, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:22.070385636Z","hostname":"buskin","pid":1661,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:22.070630973Z","hostname":"buskin","pid":1661,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:40794"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6507a4bd-9097-45d0-9787-1240feff9525, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:22.070823129Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6507a4bd-9097-45d0-9787-1240feff9525, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:22.070832939Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:22.090659158Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:22.090693219Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:22.108444126Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:22.108475576Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 416eb39a-1cea-4752-bc22-195679c44853, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:22.125628185Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 416eb39a-1cea-4752-bc22-195679c44853, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:22.125661766Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bdd869e5-e908-42b7-afb7-35517ea5ca46, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:22.176315725Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bdd869e5-e908-42b7-afb7-35517ea5ca46, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:22.176353246Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd70b00d-6277-40dc-9a7a-524663f1a340, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:22.227753189Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bd70b00d-6277-40dc-9a7a-524663f1a340, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:22.22779343Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:22.283028808Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:22.28307403Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 777ca252-0e2b-4952-8163-fd713097cfd8, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:22.334046508Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 777ca252-0e2b-4952-8163-fd713097cfd8, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:22.3340859Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:22.386872624Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:22.386903855Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:22.387162563Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:22.387294067Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:22.47184048Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:22.471879891Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:22.471888582Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:22.471903822Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:22.471910292Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:22.566464408Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_416eb39a-1cea-4752-bc22-195679c44853","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:22.57083405Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_777ca252-0e2b-4952-8163-fd713097cfd8","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:22.57513553Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_bd70b00d-6277-40dc-9a7a-524663f1a340","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:22.579623025Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_bdd869e5-e908-42b7-afb7-35517ea5ca46","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:22.583871094Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_cockroachdb_1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:22.588456242Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_416eb39a-1cea-4752-bc22-195679c44853","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:22.593211536Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_777ca252-0e2b-4952-8163-fd713097cfd8","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:22.597902888Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_bd70b00d-6277-40dc-9a7a-524663f1a340","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:22.602475896Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_bdd869e5-e908-42b7-afb7-35517ea5ca46","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:22.606672642Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_cockroachdb_1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:26.316677346Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:26.316883397Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:26.320442497Z","hostname":"buskin","pid":1661,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:776"} {"msg":"Profile for oxz_cockroachdb_416eb39a-1cea-4752-bc22-195679c44853:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:26.325134427Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:26.325227749Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_416eb39a-1cea-4752-bc22-195679c44853","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_cockroachdb_777ca252-0e2b-4952-8163-fd713097cfd8:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:26.422614705Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:26.422740108Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_777ca252-0e2b-4952-8163-fd713097cfd8","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_cockroachdb_1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b\") failed: Property not found. retry in 54.161482ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:26.464655693Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_416eb39a-1cea-4752-bc22-195679c44853\") failed: Property not found. retry in 52.133016ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:26.473093679Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_416eb39a-1cea-4752-bc22-195679c44853","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b\") failed: Property not found. retry in 131.895016ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:26.535438137Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_416eb39a-1cea-4752-bc22-195679c44853\") failed: Property not found. retry in 148.394045ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:26.640590473Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_416eb39a-1cea-4752-bc22-195679c44853","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b\") failed: Property not found. retry in 160.494995ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:26.688430222Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_777ca252-0e2b-4952-8163-fd713097cfd8\") failed: Property not found. retry in 53.876924ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:26.704415099Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_777ca252-0e2b-4952-8163-fd713097cfd8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_cockroachdb_bd70b00d-6277-40dc-9a7a-524663f1a340:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:26.708507381Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:26.708765264Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_bd70b00d-6277-40dc-9a7a-524663f1a340","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_cockroachdb_777ca252-0e2b-4952-8163-fd713097cfd8\") failed: Property not found. retry in 90.037469ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:26.785569374Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_777ca252-0e2b-4952-8163-fd713097cfd8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_cockroachdb_bdd869e5-e908-42b7-afb7-35517ea5ca46:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:26.789059271Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:26.789358971Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_bdd869e5-e908-42b7-afb7-35517ea5ca46","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_cockroachdb_416eb39a-1cea-4752-bc22-195679c44853\") failed: Property not found. retry in 258.888799ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:26.865137188Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_416eb39a-1cea-4752-bc22-195679c44853","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b\") failed: Property not found. retry in 488.62907ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:26.967374001Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_777ca252-0e2b-4952-8163-fd713097cfd8\") failed: Property not found. retry in 262.126492ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:27.036018252Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_777ca252-0e2b-4952-8163-fd713097cfd8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_416eb39a-1cea-4752-bc22-195679c44853\") failed: Property not found. retry in 552.058244ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:27.194047495Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_416eb39a-1cea-4752-bc22-195679c44853","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bd70b00d-6277-40dc-9a7a-524663f1a340\") failed: Property not found. retry in 58.349082ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:27.267168Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_bd70b00d-6277-40dc-9a7a-524663f1a340","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_777ca252-0e2b-4952-8163-fd713097cfd8\") failed: Property not found. retry in 401.740089ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:27.382729196Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_777ca252-0e2b-4952-8163-fd713097cfd8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bd70b00d-6277-40dc-9a7a-524663f1a340\") failed: Property not found. retry in 66.366908ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:27.618628463Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_bd70b00d-6277-40dc-9a7a-524663f1a340","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bdd869e5-e908-42b7-afb7-35517ea5ca46\") failed: Property not found. retry in 27.540698ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:27.682409454Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_bdd869e5-e908-42b7-afb7-35517ea5ca46","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b\") failed: Property not found. retry in 1.123077665s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:27.81939034Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_416eb39a-1cea-4752-bc22-195679c44853\") failed: Property not found. retry in 432.191733ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:28.026319864Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_416eb39a-1cea-4752-bc22-195679c44853","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_777ca252-0e2b-4952-8163-fd713097cfd8\") failed: Property not found. retry in 585.658585ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:28.104934247Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_777ca252-0e2b-4952-8163-fd713097cfd8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bd70b00d-6277-40dc-9a7a-524663f1a340\") failed: Property not found. retry in 110.520976ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:28.192712897Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_bd70b00d-6277-40dc-9a7a-524663f1a340","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bdd869e5-e908-42b7-afb7-35517ea5ca46\") failed: Property not found. retry in 78.051032ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:28.682648784Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_bdd869e5-e908-42b7-afb7-35517ea5ca46","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_416eb39a-1cea-4752-bc22-195679c44853\") failed: Property not found. retry in 1.187984851s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:28.727894214Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_416eb39a-1cea-4752-bc22-195679c44853","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_777ca252-0e2b-4952-8163-fd713097cfd8\") failed: Property not found. retry in 791.326794ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:28.809199791Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_777ca252-0e2b-4952-8163-fd713097cfd8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bd70b00d-6277-40dc-9a7a-524663f1a340\") failed: Property not found. retry in 598.166764ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:28.884260592Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_bd70b00d-6277-40dc-9a7a-524663f1a340","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bdd869e5-e908-42b7-afb7-35517ea5ca46\") failed: Property not found. retry in 176.929373ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:28.916652474Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_bdd869e5-e908-42b7-afb7-35517ea5ca46","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b\") failed: Property not found. retry in 1.415620831s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:28.959383462Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bdd869e5-e908-42b7-afb7-35517ea5ca46\") failed: Property not found. retry in 590.654112ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:29.105287333Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_bdd869e5-e908-42b7-afb7-35517ea5ca46","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bd70b00d-6277-40dc-9a7a-524663f1a340\") failed: Property not found. retry in 692.519666ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:29.493076615Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_bd70b00d-6277-40dc-9a7a-524663f1a340","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_777ca252-0e2b-4952-8163-fd713097cfd8\") failed: Property not found. retry in 690.543365ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:29.61110875Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_777ca252-0e2b-4952-8163-fd713097cfd8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bdd869e5-e908-42b7-afb7-35517ea5ca46\") failed: Property not found. retry in 1.02846694s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:29.705299444Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_bdd869e5-e908-42b7-afb7-35517ea5ca46","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_416eb39a-1cea-4752-bc22-195679c44853\") failed: Property not found. retry in 1.160727264s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:29.927515636Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_416eb39a-1cea-4752-bc22-195679c44853","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bd70b00d-6277-40dc-9a7a-524663f1a340\") failed: Property not found. retry in 822.842063ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:30.197553072Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_bd70b00d-6277-40dc-9a7a-524663f1a340","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_777ca252-0e2b-4952-8163-fd713097cfd8\") failed: Property not found. retry in 788.342707ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:30.311749031Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_777ca252-0e2b-4952-8163-fd713097cfd8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b\") failed: Property not found. retry in 1.220311817s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:30.386070366Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bdd869e5-e908-42b7-afb7-35517ea5ca46\") failed: Property not found. retry in 927.982262ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:30.747408329Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_bdd869e5-e908-42b7-afb7-35517ea5ca46","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bd70b00d-6277-40dc-9a7a-524663f1a340\") failed: Property not found. retry in 842.728709ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:31.034698235Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_bd70b00d-6277-40dc-9a7a-524663f1a340","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_416eb39a-1cea-4752-bc22-195679c44853\") failed: Property not found. retry in 1.022773846s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:31.101770551Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_416eb39a-1cea-4752-bc22-195679c44853","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_777ca252-0e2b-4952-8163-fd713097cfd8\") failed: Property not found. retry in 942.478953ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:31.113359581Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_777ca252-0e2b-4952-8163-fd713097cfd8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b\") failed: Property not found. retry in 675.35356ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:31.616870408Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bdd869e5-e908-42b7-afb7-35517ea5ca46\") failed: Property not found. retry in 960.377877ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:31.68513937Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_bdd869e5-e908-42b7-afb7-35517ea5ca46","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bd70b00d-6277-40dc-9a7a-524663f1a340\") failed: Property not found. retry in 999.327736ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:31.887732618Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_bd70b00d-6277-40dc-9a7a-524663f1a340","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_777ca252-0e2b-4952-8163-fd713097cfd8\") failed: Property not found. retry in 1.024832075s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:32.064398294Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_777ca252-0e2b-4952-8163-fd713097cfd8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_416eb39a-1cea-4752-bc22-195679c44853\") failed: Property not found. retry in 888.912634ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:32.133665286Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_416eb39a-1cea-4752-bc22-195679c44853","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b\") failed: Property not found. retry in 694.881605ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:32.301238998Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bdd869e5-e908-42b7-afb7-35517ea5ca46\") failed: Property not found. retry in 890.475988ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:32.654771565Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_bdd869e5-e908-42b7-afb7-35517ea5ca46","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bd70b00d-6277-40dc-9a7a-524663f1a340\") failed: Property not found. retry in 656.819001ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:32.895480855Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_bd70b00d-6277-40dc-9a7a-524663f1a340","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b\") failed: Property not found. retry in 842.604034ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:33.00557623Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_416eb39a-1cea-4752-bc22-195679c44853\") failed: Property not found. retry in 609.110032ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:33.034187184Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_416eb39a-1cea-4752-bc22-195679c44853","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_777ca252-0e2b-4952-8163-fd713097cfd8\") failed: Property not found. retry in 704.126261ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:33.100025433Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_777ca252-0e2b-4952-8163-fd713097cfd8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bdd869e5-e908-42b7-afb7-35517ea5ca46\") failed: Property not found. retry in 520.161739ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:33.555909561Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_bdd869e5-e908-42b7-afb7-35517ea5ca46","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bd70b00d-6277-40dc-9a7a-524663f1a340\") failed: Property not found. retry in 804.852992ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:33.565571192Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_bd70b00d-6277-40dc-9a7a-524663f1a340","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_416eb39a-1cea-4752-bc22-195679c44853\") failed: Property not found. retry in 512.513394ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:33.654238841Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_416eb39a-1cea-4752-bc22-195679c44853","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_777ca252-0e2b-4952-8163-fd713097cfd8\") failed: Property not found. retry in 1.025642372s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:33.81480789Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_777ca252-0e2b-4952-8163-fd713097cfd8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b\") failed: Property not found. retry in 1.27671278s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:33.859650594Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bdd869e5-e908-42b7-afb7-35517ea5ca46\") failed: Property not found. retry in 1.317855362s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:34.087861817Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_bdd869e5-e908-42b7-afb7-35517ea5ca46","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bd70b00d-6277-40dc-9a7a-524663f1a340\") failed: Property not found. retry in 724.286541ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:34.380771694Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_bd70b00d-6277-40dc-9a7a-524663f1a340","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bd70b00d-6277-40dc-9a7a-524663f1a340\") failed: Property not found. retry in 1.145994751s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:35.115241976Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_bd70b00d-6277-40dc-9a7a-524663f1a340","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b\") failed: Property not found. retry in 923.390228ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:35.147659515Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bdd869e5-e908-42b7-afb7-35517ea5ca46\") failed: Property not found. retry in 1.324104294s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:35.417131724Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_bdd869e5-e908-42b7-afb7-35517ea5ca46","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b\") failed: Property not found. retry in 1.335615125s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:36.080831829Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bd70b00d-6277-40dc-9a7a-524663f1a340\") failed: Property not found. retry in 756.451508ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:36.2715845Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_bd70b00d-6277-40dc-9a7a-524663f1a340","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bdd869e5-e908-42b7-afb7-35517ea5ca46\") failed: Property not found. retry in 1.043618046s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:37.002230807Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_bdd869e5-e908-42b7-afb7-35517ea5ca46","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bd70b00d-6277-40dc-9a7a-524663f1a340\") failed: Property not found. retry in 637.489105ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:37.038972457Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_bd70b00d-6277-40dc-9a7a-524663f1a340","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:37.178124659Z","hostname":"buskin","pid":1661,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:34889"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:37.17912565Z","hostname":"buskin","pid":1661,"uri":"/baseboard","method":"GET","req_id":"039382ed-dee1-4425-98ac-165aeb4a599c","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:34889","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":929,"response_code":"200"} {"msg":"wait for service Some(\"oxz_cockroachdb_1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b\") failed: Property not found. retry in 1.484545393s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:37.437137272Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bd70b00d-6277-40dc-9a7a-524663f1a340\") failed: Property not found. retry in 618.751394ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:37.690366Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_bd70b00d-6277-40dc-9a7a-524663f1a340","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bdd869e5-e908-42b7-afb7-35517ea5ca46\") failed: Property not found. retry in 888.100303ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:38.056686434Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_bdd869e5-e908-42b7-afb7-35517ea5ca46","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bd70b00d-6277-40dc-9a7a-524663f1a340\") failed: Property not found. retry in 1.386591859s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:38.318297825Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_bd70b00d-6277-40dc-9a7a-524663f1a340","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b\") failed: Property not found. retry in 1.417036801s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:39.019267596Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bdd869e5-e908-42b7-afb7-35517ea5ca46\") failed: Property not found. retry in 1.495416045s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:39.03495777Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_bdd869e5-e908-42b7-afb7-35517ea5ca46","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bd70b00d-6277-40dc-9a7a-524663f1a340\") failed: Property not found. retry in 934.563277ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:39.717292417Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_bd70b00d-6277-40dc-9a7a-524663f1a340","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b\") failed: Property not found. retry in 1.247408964s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:40.446406188Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bdd869e5-e908-42b7-afb7-35517ea5ca46\") failed: Property not found. retry in 1.423454208s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:40.539289663Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_bdd869e5-e908-42b7-afb7-35517ea5ca46","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bd70b00d-6277-40dc-9a7a-524663f1a340\") failed: Property not found. retry in 1.470009588s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:40.660288587Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_bd70b00d-6277-40dc-9a7a-524663f1a340","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b\") failed: Property not found. retry in 618.14855ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:41.702264258Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bdd869e5-e908-42b7-afb7-35517ea5ca46\") failed: Property not found. retry in 1.044229886s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:41.971474308Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_bdd869e5-e908-42b7-afb7-35517ea5ca46","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bd70b00d-6277-40dc-9a7a-524663f1a340\") failed: Property not found. retry in 756.827645ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:42.139683489Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_bd70b00d-6277-40dc-9a7a-524663f1a340","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b\") failed: Property not found. retry in 1.336990994s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:42.328919664Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bd70b00d-6277-40dc-9a7a-524663f1a340\") failed: Property not found. retry in 857.412002ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:42.905422986Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_bd70b00d-6277-40dc-9a7a-524663f1a340","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bdd869e5-e908-42b7-afb7-35517ea5ca46\") failed: Property not found. retry in 808.217567ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:43.024004007Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_bdd869e5-e908-42b7-afb7-35517ea5ca46","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b\") failed: Property not found. retry in 1.202254824s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:43.674497363Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bd70b00d-6277-40dc-9a7a-524663f1a340\") failed: Property not found. retry in 1.307580623s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:43.770954936Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_bd70b00d-6277-40dc-9a7a-524663f1a340","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bdd869e5-e908-42b7-afb7-35517ea5ca46\") failed: Property not found. retry in 774.815718ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:43.841243379Z","hostname":"buskin","pid":1661,"zone":"oxz_cockroachdb_bdd869e5-e908-42b7-afb7-35517ea5ca46","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:45.099083819Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2908","zone":"oxz_cockroachdb_1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:45.099141781Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2908","zone":"oxz_cockroachdb_416eb39a-1cea-4752-bc22-195679c44853"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:45.099152481Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2908","zone":"oxz_cockroachdb_777ca252-0e2b-4952-8163-fd713097cfd8"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:45.099157671Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2908","zone":"oxz_cockroachdb_bd70b00d-6277-40dc-9a7a-524663f1a340"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:45.099162191Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2908","zone":"oxz_cockroachdb_bdd869e5-e908-42b7-afb7-35517ea5ca46"} {"msg":"Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:45.099170121Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:198"} {"msg":"Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:45.099485181Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:198"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:45.099727798Z","hostname":"buskin","pid":1661,"uri":"/omicron-zones","method":"PUT","req_id":"24841534-564e-4ac3-8f70-3f76cc25324d","remote_addr":"[fd00:1122:3344:101::1]:40794","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":22779253,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:45.135428306Z","hostname":"buskin","pid":1661,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:53164"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:45.165619048Z","hostname":"buskin","pid":1661,"uri":"/cockroachdb","method":"POST","req_id":"2a407ec1-2f6d-4907-af25-dac5d71c5e65","remote_addr":"[fd00:1122:3344:101::1]:53164","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService5/omicron6 in oxz_cockroachdb_1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b /usr/sbin/ipadm show-addr -p -o ADDR oxControlService5/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":30069,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:45.165935998Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:755","retry_after":"130.817699ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2a407ec1-2f6d-4907-af25-dac5d71c5e65\", \"content-length\": \"124\", \"date\": \"Thu, 07 Mar 2024 22:44:45 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"2a407ec1-2f6d-4907-af25-dac5d71c5e65\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:45.324147706Z","hostname":"buskin","pid":1661,"uri":"/cockroachdb","method":"POST","req_id":"904f59d3-c103-442a-84b8-370e2cb2be2f","remote_addr":"[fd00:1122:3344:101::1]:53164","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService5/omicron6 in oxz_cockroachdb_1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b /usr/sbin/ipadm show-addr -p -o ADDR oxControlService5/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":26696,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:44:45.324498597Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:755","retry_after":"171.697551ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"904f59d3-c103-442a-84b8-370e2cb2be2f\", \"content-length\": \"124\", \"date\": \"Thu, 07 Mar 2024 22:44:45 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"904f59d3-c103-442a-84b8-370e2cb2be2f\" }"} {"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::6]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:44:45.526525069Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:3314"} {"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:01.949083992Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:3332"} {"msg":"Formatting CRDB - Completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:13.04661987Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:3355"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:13.047039143Z","hostname":"buskin","pid":1661,"uri":"/cockroachdb","method":"POST","req_id":"d6829611-63b9-429f-8aa6-54c013dedc54","remote_addr":"[fd00:1122:3344:101::1]:53164","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":27548979,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:13.048102415Z","hostname":"buskin","pid":1661,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:56282"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:13.04891278Z","hostname":"buskin","pid":1661,"uri":"/baseboard","method":"GET","req_id":"d8f42eed-322b-4b0e-a468-0e3290705e5d","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:56282","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":372,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 6507a4bd-9097-45d0-9787-1240feff9525, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 65847932-72ba-40f7-b62c-5adce2529400, kind: Service { id: f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf }, name: Name(\"ntp-f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 416eb39a-1cea-4752-bc22-195679c44853, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bdd869e5-e908-42b7-afb7-35517ea5ca46, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bd70b00d-6277-40dc-9a7a-524663f1a340, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 777ca252-0e2b-4952-8163-fd713097cfd8, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 0b3b3eae-a95f-4091-984a-93fccf167e79, kind: Service { id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c }, name: Name(\"external-dns-e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: ca9c5211-aa62-4394-a496-e20bf51739bc, kind: Service { id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b }, name: Name(\"external-dns-472c8e8c-ac8f-4193-921c-49dd3ceb288b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 761d7d83-4f44-4dad-ba8f-694aeae46e71, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 1eeeae19-992a-47f8-9e7b-dc93c05f0958, kind: Service { id: 761d7d83-4f44-4dad-ba8f-694aeae46e71 }, name: Name(\"nexus-761d7d83-4f44-4dad-ba8f-694aeae46e71\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a835148-a2f0-40d4-bc66-05f1d779465b, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 89eadaf9-099e-4929-ad88-e7ee2258e13d, kind: Service { id: 0a835148-a2f0-40d4-bc66-05f1d779465b }, name: Name(\"nexus-0a835148-a2f0-40d4-bc66-05f1d779465b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 72])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2bea071a-6ee2-45ba-9a69-93a0e2e5b31b, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 8c296dc4-9b38-40c8-9549-80b2967f6cdc, kind: Service { id: 2bea071a-6ee2-45ba-9a69-93a0e2e5b31b }, name: Name(\"nexus-2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8dc5039d-cb88-49dd-8ab8-2ca47e481b18, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: e79fbd89-0714-4c5d-b550-de2def94225f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 16f9d651-64fc-4e3c-a69b-bda36e263e78, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: afeebd42-9418-4a17-adf1-4ce6f3cadd20, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 5eefa476-ecad-43d0-a297-e34591893cf5, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a18cc15b-758a-467a-8a3b-ed115f6e1135, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 36c43d98-b274-450d-98d0-db7e0ee7514c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 9227e547-c76f-4bc5-b6f2-101d008d915e, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 608a3190-2550-47a2-a23f-81797b571827, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: fca244c4-2682-42f7-9297-53c69ba4caa1, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:13.106041575Z","hostname":"buskin","pid":1661,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:13.106851679Z","hostname":"buskin","pid":1661,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:58669"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6507a4bd-9097-45d0-9787-1240feff9525, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:13.108299413Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6507a4bd-9097-45d0-9787-1240feff9525, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:13.108311824Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:13.132820604Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:13.132883756Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:13.152592031Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:13.154988133Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 416eb39a-1cea-4752-bc22-195679c44853, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:13.171610695Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 416eb39a-1cea-4752-bc22-195679c44853, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:13.171675677Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bdd869e5-e908-42b7-afb7-35517ea5ca46, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:13.191166826Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bdd869e5-e908-42b7-afb7-35517ea5ca46, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:13.191219528Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd70b00d-6277-40dc-9a7a-524663f1a340, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:13.210017595Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bd70b00d-6277-40dc-9a7a-524663f1a340, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:13.210071287Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:13.229518964Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:13.229568426Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 777ca252-0e2b-4952-8163-fd713097cfd8, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:13.247624711Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 777ca252-0e2b-4952-8163-fd713097cfd8, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:13.247676343Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:13.265021267Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:13.265070578Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:13.41181821Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:13.411872382Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:13.566377188Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:13.56645043Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:13.697876879Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:13.697942131Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5eefa476-ecad-43d0-a297-e34591893cf5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:13.833088783Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5eefa476-ecad-43d0-a297-e34591893cf5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:13.833154375Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a18cc15b-758a-467a-8a3b-ed115f6e1135, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:13.944492768Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a18cc15b-758a-467a-8a3b-ed115f6e1135, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:13.94456257Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 36c43d98-b274-450d-98d0-db7e0ee7514c, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:14.131270399Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 36c43d98-b274-450d-98d0-db7e0ee7514c, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:14.131339161Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9227e547-c76f-4bc5-b6f2-101d008d915e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:14.34058144Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9227e547-c76f-4bc5-b6f2-101d008d915e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:14.340643762Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:14.462214224Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:14.462269136Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 608a3190-2550-47a2-a23f-81797b571827, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:14.61703547Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 608a3190-2550-47a2-a23f-81797b571827, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:14.617091822Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:14.744876781Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:14.744938543Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fca244c4-2682-42f7-9297-53c69ba4caa1, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:14.891890451Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fca244c4-2682-42f7-9297-53c69ba4caa1, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:14.891944843Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:15.077143066Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:15.077212298Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:15.077631881Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:15.077832077Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:15.180973872Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:15.181012233Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:15.181039264Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:15.181045574Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:15.181053375Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:15.181059015Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:15.181065315Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:15.181071325Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:15.181076895Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:15.181082736Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:15.181088266Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:15.181095346Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:15.181101146Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:15.181106926Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:15.181113917Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:15.181119537Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:15.181124727Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:15.181128777Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:15.181143157Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:15.18122652Z","hostname":"buskin","pid":1661,"uri":"/omicron-zones","method":"PUT","req_id":"40431e4c-0ea9-49c2-85c5-318e2a739b84","remote_addr":"[fd00:1122:3344:101::1]:58669","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b\", TimeNotSynchronized), (\"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135\", TimeNotSynchronized), (\"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f\", TimeNotSynchronized), (\"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71\", TimeNotSynchronized), (\"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b\", TimeNotSynchronized), (\"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78\", TimeNotSynchronized), (\"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c\", TimeNotSynchronized), (\"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a\", TimeNotSynchronized), (\"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a\", TimeNotSynchronized), (\"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827\", TimeNotSynchronized), (\"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4\", TimeNotSynchronized), (\"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\", TimeNotSynchronized), (\"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b\", TimeNotSynchronized), (\"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5\", TimeNotSynchronized), (\"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18\", TimeNotSynchronized), (\"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20\", TimeNotSynchronized), (\"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e\", TimeNotSynchronized), (\"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\", TimeNotSynchronized), (\"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1\", TimeNotSynchronized)]","latency_us":2073423,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:15.181700934Z","hostname":"buskin","pid":1661,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"103.01908ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"40431e4c-0ea9-49c2-85c5-318e2a739b84\", \"content-length\": \"124\", \"date\": \"Thu, 07 Mar 2024 22:45:15 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"40431e4c-0ea9-49c2-85c5-318e2a739b84\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 6507a4bd-9097-45d0-9787-1240feff9525, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 65847932-72ba-40f7-b62c-5adce2529400, kind: Service { id: f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf }, name: Name(\"ntp-f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 416eb39a-1cea-4752-bc22-195679c44853, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bdd869e5-e908-42b7-afb7-35517ea5ca46, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bd70b00d-6277-40dc-9a7a-524663f1a340, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 777ca252-0e2b-4952-8163-fd713097cfd8, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 0b3b3eae-a95f-4091-984a-93fccf167e79, kind: Service { id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c }, name: Name(\"external-dns-e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: ca9c5211-aa62-4394-a496-e20bf51739bc, kind: Service { id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b }, name: Name(\"external-dns-472c8e8c-ac8f-4193-921c-49dd3ceb288b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 761d7d83-4f44-4dad-ba8f-694aeae46e71, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 1eeeae19-992a-47f8-9e7b-dc93c05f0958, kind: Service { id: 761d7d83-4f44-4dad-ba8f-694aeae46e71 }, name: Name(\"nexus-761d7d83-4f44-4dad-ba8f-694aeae46e71\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a835148-a2f0-40d4-bc66-05f1d779465b, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 89eadaf9-099e-4929-ad88-e7ee2258e13d, kind: Service { id: 0a835148-a2f0-40d4-bc66-05f1d779465b }, name: Name(\"nexus-0a835148-a2f0-40d4-bc66-05f1d779465b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 72])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2bea071a-6ee2-45ba-9a69-93a0e2e5b31b, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 8c296dc4-9b38-40c8-9549-80b2967f6cdc, kind: Service { id: 2bea071a-6ee2-45ba-9a69-93a0e2e5b31b }, name: Name(\"nexus-2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8dc5039d-cb88-49dd-8ab8-2ca47e481b18, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: e79fbd89-0714-4c5d-b550-de2def94225f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 16f9d651-64fc-4e3c-a69b-bda36e263e78, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: afeebd42-9418-4a17-adf1-4ce6f3cadd20, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 5eefa476-ecad-43d0-a297-e34591893cf5, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a18cc15b-758a-467a-8a3b-ed115f6e1135, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 36c43d98-b274-450d-98d0-db7e0ee7514c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 9227e547-c76f-4bc5-b6f2-101d008d915e, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 608a3190-2550-47a2-a23f-81797b571827, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: fca244c4-2682-42f7-9297-53c69ba4caa1, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:15.285803048Z","hostname":"buskin","pid":1661,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6507a4bd-9097-45d0-9787-1240feff9525, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:15.286431097Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6507a4bd-9097-45d0-9787-1240feff9525, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:15.286442248Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:15.308020679Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:15.308088191Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:15.328122166Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:15.328188219Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 416eb39a-1cea-4752-bc22-195679c44853, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:15.347216253Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 416eb39a-1cea-4752-bc22-195679c44853, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:15.347268045Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bdd869e5-e908-42b7-afb7-35517ea5ca46, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:15.36531942Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bdd869e5-e908-42b7-afb7-35517ea5ca46, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:15.365369922Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd70b00d-6277-40dc-9a7a-524663f1a340, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:15.38418868Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bd70b00d-6277-40dc-9a7a-524663f1a340, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:15.384242421Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:15.403277956Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:15.403342588Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 777ca252-0e2b-4952-8163-fd713097cfd8, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:15.421880168Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 777ca252-0e2b-4952-8163-fd713097cfd8, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:15.42193008Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:15.44082719Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:15.440885752Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:15.460085912Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:15.460134873Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:15.479332593Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:15.479382365Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:15.497611396Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:15.497660657Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5eefa476-ecad-43d0-a297-e34591893cf5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:15.515538017Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5eefa476-ecad-43d0-a297-e34591893cf5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:15.515597139Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a18cc15b-758a-467a-8a3b-ed115f6e1135, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:15.533821469Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a18cc15b-758a-467a-8a3b-ed115f6e1135, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:15.533872361Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 36c43d98-b274-450d-98d0-db7e0ee7514c, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:15.553328888Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 36c43d98-b274-450d-98d0-db7e0ee7514c, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:15.553376669Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9227e547-c76f-4bc5-b6f2-101d008d915e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:15.57193972Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9227e547-c76f-4bc5-b6f2-101d008d915e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:15.571992401Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:15.590685156Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:15.590733867Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 608a3190-2550-47a2-a23f-81797b571827, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:15.608770492Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 608a3190-2550-47a2-a23f-81797b571827, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:15.608843634Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:15.626929261Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:15.626976792Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fca244c4-2682-42f7-9297-53c69ba4caa1, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:15.64613141Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fca244c4-2682-42f7-9297-53c69ba4caa1, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:15.646185012Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:15.66399459Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:15.664055992Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:15.664276749Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:15.664404102Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:15.758649549Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:15.758710031Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:15.758718721Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:15.758724831Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:15.758731781Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:15.758738051Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:15.758750612Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:15.758757392Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:15.758765102Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:15.758771103Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:15.758776633Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:15.758784543Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:15.758789193Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:15.758796423Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:15.758801704Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:15.758826014Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:15.758831694Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:15.758837605Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:15.758845015Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:15.758978069Z","hostname":"buskin","pid":1661,"uri":"/omicron-zones","method":"PUT","req_id":"91a1b057-a24c-417f-b5f1-8ceb8a24bf34","remote_addr":"[fd00:1122:3344:101::1]:58669","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\", TimeNotSynchronized), (\"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5\", TimeNotSynchronized), (\"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135\", TimeNotSynchronized), (\"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4\", TimeNotSynchronized), (\"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c\", TimeNotSynchronized), (\"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78\", TimeNotSynchronized), (\"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827\", TimeNotSynchronized), (\"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1\", TimeNotSynchronized), (\"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b\", TimeNotSynchronized), (\"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18\", TimeNotSynchronized), (\"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b\", TimeNotSynchronized), (\"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a\", TimeNotSynchronized), (\"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b\", TimeNotSynchronized), (\"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71\", TimeNotSynchronized), (\"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f\", TimeNotSynchronized), (\"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e\", TimeNotSynchronized), (\"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20\", TimeNotSynchronized), (\"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a\", TimeNotSynchronized), (\"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\", TimeNotSynchronized)]","latency_us":472764,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:15.759232057Z","hostname":"buskin","pid":1661,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"155.896346ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"91a1b057-a24c-417f-b5f1-8ceb8a24bf34\", \"content-length\": \"124\", \"date\": \"Thu, 07 Mar 2024 22:45:15 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"91a1b057-a24c-417f-b5f1-8ceb8a24bf34\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 6507a4bd-9097-45d0-9787-1240feff9525, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 65847932-72ba-40f7-b62c-5adce2529400, kind: Service { id: f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf }, name: Name(\"ntp-f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 416eb39a-1cea-4752-bc22-195679c44853, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bdd869e5-e908-42b7-afb7-35517ea5ca46, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bd70b00d-6277-40dc-9a7a-524663f1a340, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 777ca252-0e2b-4952-8163-fd713097cfd8, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 0b3b3eae-a95f-4091-984a-93fccf167e79, kind: Service { id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c }, name: Name(\"external-dns-e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: ca9c5211-aa62-4394-a496-e20bf51739bc, kind: Service { id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b }, name: Name(\"external-dns-472c8e8c-ac8f-4193-921c-49dd3ceb288b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 761d7d83-4f44-4dad-ba8f-694aeae46e71, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 1eeeae19-992a-47f8-9e7b-dc93c05f0958, kind: Service { id: 761d7d83-4f44-4dad-ba8f-694aeae46e71 }, name: Name(\"nexus-761d7d83-4f44-4dad-ba8f-694aeae46e71\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a835148-a2f0-40d4-bc66-05f1d779465b, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 89eadaf9-099e-4929-ad88-e7ee2258e13d, kind: Service { id: 0a835148-a2f0-40d4-bc66-05f1d779465b }, name: Name(\"nexus-0a835148-a2f0-40d4-bc66-05f1d779465b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 72])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2bea071a-6ee2-45ba-9a69-93a0e2e5b31b, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 8c296dc4-9b38-40c8-9549-80b2967f6cdc, kind: Service { id: 2bea071a-6ee2-45ba-9a69-93a0e2e5b31b }, name: Name(\"nexus-2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8dc5039d-cb88-49dd-8ab8-2ca47e481b18, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: e79fbd89-0714-4c5d-b550-de2def94225f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 16f9d651-64fc-4e3c-a69b-bda36e263e78, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: afeebd42-9418-4a17-adf1-4ce6f3cadd20, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 5eefa476-ecad-43d0-a297-e34591893cf5, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a18cc15b-758a-467a-8a3b-ed115f6e1135, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 36c43d98-b274-450d-98d0-db7e0ee7514c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 9227e547-c76f-4bc5-b6f2-101d008d915e, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 608a3190-2550-47a2-a23f-81797b571827, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: fca244c4-2682-42f7-9297-53c69ba4caa1, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:15.917127085Z","hostname":"buskin","pid":1661,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6507a4bd-9097-45d0-9787-1240feff9525, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:15.917872348Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6507a4bd-9097-45d0-9787-1240feff9525, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:15.917886838Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:15.940626125Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:15.940677386Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:15.961491625Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:15.964024892Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 416eb39a-1cea-4752-bc22-195679c44853, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:15.980456618Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 416eb39a-1cea-4752-bc22-195679c44853, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:15.98050666Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bdd869e5-e908-42b7-afb7-35517ea5ca46, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:15.999342198Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bdd869e5-e908-42b7-afb7-35517ea5ca46, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:15.9993907Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd70b00d-6277-40dc-9a7a-524663f1a340, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.02124216Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bd70b00d-6277-40dc-9a7a-524663f1a340, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.021306352Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.042392099Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.04244611Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 777ca252-0e2b-4952-8163-fd713097cfd8, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.063816996Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 777ca252-0e2b-4952-8163-fd713097cfd8, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.063872497Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.083571062Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.083622084Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.103829004Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.103887426Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.123442037Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.123490418Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.142892624Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.143028028Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5eefa476-ecad-43d0-a297-e34591893cf5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.163385993Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5eefa476-ecad-43d0-a297-e34591893cf5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.163444495Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a18cc15b-758a-467a-8a3b-ed115f6e1135, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.182252723Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a18cc15b-758a-467a-8a3b-ed115f6e1135, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.182307664Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 36c43d98-b274-450d-98d0-db7e0ee7514c, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.201478814Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 36c43d98-b274-450d-98d0-db7e0ee7514c, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.201531505Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9227e547-c76f-4bc5-b6f2-101d008d915e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.220407955Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9227e547-c76f-4bc5-b6f2-101d008d915e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.220464327Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.239579584Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.239643156Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 608a3190-2550-47a2-a23f-81797b571827, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.258175326Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 608a3190-2550-47a2-a23f-81797b571827, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.258228217Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.277340275Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.279872921Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fca244c4-2682-42f7-9297-53c69ba4caa1, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.296187414Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fca244c4-2682-42f7-9297-53c69ba4caa1, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.296239976Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.315122286Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.315207448Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.315427625Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.31558187Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:16.412191398Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:16.4122547Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:16.41226438Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:16.41227361Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:16.41228254Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:16.41228929Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:16.412297881Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:16.412306431Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:16.412314721Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:16.412321691Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:16.412329982Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:16.412338342Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:16.412365703Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:16.412372313Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:16.412378883Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:16.412384213Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:16.412389533Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:16.412394814Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:16.412407104Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.412480656Z","hostname":"buskin","pid":1661,"uri":"/omicron-zones","method":"PUT","req_id":"78e4b54d-365b-4c2d-a6b7-160a7284292f","remote_addr":"[fd00:1122:3344:101::1]:58669","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20\", TimeNotSynchronized), (\"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827\", TimeNotSynchronized), (\"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71\", TimeNotSynchronized), (\"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\", TimeNotSynchronized), (\"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b\", TimeNotSynchronized), (\"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f\", TimeNotSynchronized), (\"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78\", TimeNotSynchronized), (\"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18\", TimeNotSynchronized), (\"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4\", TimeNotSynchronized), (\"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5\", TimeNotSynchronized), (\"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135\", TimeNotSynchronized), (\"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c\", TimeNotSynchronized), (\"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b\", TimeNotSynchronized), (\"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b\", TimeNotSynchronized), (\"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e\", TimeNotSynchronized), (\"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a\", TimeNotSynchronized), (\"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1\", TimeNotSynchronized), (\"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a\", TimeNotSynchronized), (\"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\", TimeNotSynchronized)]","latency_us":494840,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:16.412759045Z","hostname":"buskin","pid":1661,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"116.833056ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"78e4b54d-365b-4c2d-a6b7-160a7284292f\", \"content-length\": \"124\", \"date\": \"Thu, 07 Mar 2024 22:45:16 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"78e4b54d-365b-4c2d-a6b7-160a7284292f\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 6507a4bd-9097-45d0-9787-1240feff9525, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 65847932-72ba-40f7-b62c-5adce2529400, kind: Service { id: f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf }, name: Name(\"ntp-f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 416eb39a-1cea-4752-bc22-195679c44853, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bdd869e5-e908-42b7-afb7-35517ea5ca46, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bd70b00d-6277-40dc-9a7a-524663f1a340, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 777ca252-0e2b-4952-8163-fd713097cfd8, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 0b3b3eae-a95f-4091-984a-93fccf167e79, kind: Service { id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c }, name: Name(\"external-dns-e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: ca9c5211-aa62-4394-a496-e20bf51739bc, kind: Service { id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b }, name: Name(\"external-dns-472c8e8c-ac8f-4193-921c-49dd3ceb288b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 761d7d83-4f44-4dad-ba8f-694aeae46e71, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 1eeeae19-992a-47f8-9e7b-dc93c05f0958, kind: Service { id: 761d7d83-4f44-4dad-ba8f-694aeae46e71 }, name: Name(\"nexus-761d7d83-4f44-4dad-ba8f-694aeae46e71\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a835148-a2f0-40d4-bc66-05f1d779465b, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 89eadaf9-099e-4929-ad88-e7ee2258e13d, kind: Service { id: 0a835148-a2f0-40d4-bc66-05f1d779465b }, name: Name(\"nexus-0a835148-a2f0-40d4-bc66-05f1d779465b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 72])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2bea071a-6ee2-45ba-9a69-93a0e2e5b31b, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 8c296dc4-9b38-40c8-9549-80b2967f6cdc, kind: Service { id: 2bea071a-6ee2-45ba-9a69-93a0e2e5b31b }, name: Name(\"nexus-2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8dc5039d-cb88-49dd-8ab8-2ca47e481b18, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: e79fbd89-0714-4c5d-b550-de2def94225f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 16f9d651-64fc-4e3c-a69b-bda36e263e78, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: afeebd42-9418-4a17-adf1-4ce6f3cadd20, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 5eefa476-ecad-43d0-a297-e34591893cf5, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a18cc15b-758a-467a-8a3b-ed115f6e1135, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 36c43d98-b274-450d-98d0-db7e0ee7514c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 9227e547-c76f-4bc5-b6f2-101d008d915e, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 608a3190-2550-47a2-a23f-81797b571827, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: fca244c4-2682-42f7-9297-53c69ba4caa1, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.530303865Z","hostname":"buskin","pid":1661,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6507a4bd-9097-45d0-9787-1240feff9525, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.531007626Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6507a4bd-9097-45d0-9787-1240feff9525, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.531019967Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.552319789Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.55236832Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.572452947Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.572500848Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 416eb39a-1cea-4752-bc22-195679c44853, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.591212014Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 416eb39a-1cea-4752-bc22-195679c44853, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.591261335Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bdd869e5-e908-42b7-afb7-35517ea5ca46, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.610872378Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bdd869e5-e908-42b7-afb7-35517ea5ca46, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.610920059Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd70b00d-6277-40dc-9a7a-524663f1a340, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.630520561Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bd70b00d-6277-40dc-9a7a-524663f1a340, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.630572753Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.649671079Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.649717551Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 777ca252-0e2b-4952-8163-fd713097cfd8, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.66757123Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 777ca252-0e2b-4952-8163-fd713097cfd8, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.667617491Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.686088639Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.686151751Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.705165335Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.705213867Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.725341635Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.725390316Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.744211565Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.744264136Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5eefa476-ecad-43d0-a297-e34591893cf5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.762472726Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5eefa476-ecad-43d0-a297-e34591893cf5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.762525728Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a18cc15b-758a-467a-8a3b-ed115f6e1135, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.781944634Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a18cc15b-758a-467a-8a3b-ed115f6e1135, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.781992326Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 36c43d98-b274-450d-98d0-db7e0ee7514c, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.800763323Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 36c43d98-b274-450d-98d0-db7e0ee7514c, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.800810654Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9227e547-c76f-4bc5-b6f2-101d008d915e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.819645073Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9227e547-c76f-4bc5-b6f2-101d008d915e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.819695945Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.837896974Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.837945196Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 608a3190-2550-47a2-a23f-81797b571827, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.856712412Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 608a3190-2550-47a2-a23f-81797b571827, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.856758704Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.876704006Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.876750998Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fca244c4-2682-42f7-9297-53c69ba4caa1, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.895795593Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fca244c4-2682-42f7-9297-53c69ba4caa1, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.895857655Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.914720024Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.914792537Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.915150217Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:16.915319082Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:17.014636512Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:17.014699634Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:17.014713104Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:17.014719025Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:17.014724835Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:17.014732075Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:17.014737295Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:17.014744025Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:17.014751096Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:17.014777597Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:17.014785487Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:17.014793847Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:17.014802447Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:17.014807607Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:17.014813938Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:17.014819968Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:17.014824908Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:17.014829628Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:17.014836908Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:17.014909541Z","hostname":"buskin","pid":1661,"uri":"/omicron-zones","method":"PUT","req_id":"4a0cf61a-e86d-48ff-a097-800e70f8a422","remote_addr":"[fd00:1122:3344:101::1]:58669","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\", TimeNotSynchronized), (\"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18\", TimeNotSynchronized), (\"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b\", TimeNotSynchronized), (\"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e\", TimeNotSynchronized), (\"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827\", TimeNotSynchronized), (\"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a\", TimeNotSynchronized), (\"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4\", TimeNotSynchronized), (\"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5\", TimeNotSynchronized), (\"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f\", TimeNotSynchronized), (\"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20\", TimeNotSynchronized), (\"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1\", TimeNotSynchronized), (\"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b\", TimeNotSynchronized), (\"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c\", TimeNotSynchronized), (\"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b\", TimeNotSynchronized), (\"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71\", TimeNotSynchronized), (\"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\", TimeNotSynchronized), (\"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78\", TimeNotSynchronized), (\"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a\", TimeNotSynchronized), (\"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135\", TimeNotSynchronized)]","latency_us":484168,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:17.015177858Z","hostname":"buskin","pid":1661,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"168.664576ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4a0cf61a-e86d-48ff-a097-800e70f8a422\", \"content-length\": \"124\", \"date\": \"Thu, 07 Mar 2024 22:45:16 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"4a0cf61a-e86d-48ff-a097-800e70f8a422\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 6507a4bd-9097-45d0-9787-1240feff9525, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 65847932-72ba-40f7-b62c-5adce2529400, kind: Service { id: f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf }, name: Name(\"ntp-f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 416eb39a-1cea-4752-bc22-195679c44853, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bdd869e5-e908-42b7-afb7-35517ea5ca46, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bd70b00d-6277-40dc-9a7a-524663f1a340, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 777ca252-0e2b-4952-8163-fd713097cfd8, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 0b3b3eae-a95f-4091-984a-93fccf167e79, kind: Service { id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c }, name: Name(\"external-dns-e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: ca9c5211-aa62-4394-a496-e20bf51739bc, kind: Service { id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b }, name: Name(\"external-dns-472c8e8c-ac8f-4193-921c-49dd3ceb288b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 761d7d83-4f44-4dad-ba8f-694aeae46e71, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 1eeeae19-992a-47f8-9e7b-dc93c05f0958, kind: Service { id: 761d7d83-4f44-4dad-ba8f-694aeae46e71 }, name: Name(\"nexus-761d7d83-4f44-4dad-ba8f-694aeae46e71\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a835148-a2f0-40d4-bc66-05f1d779465b, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 89eadaf9-099e-4929-ad88-e7ee2258e13d, kind: Service { id: 0a835148-a2f0-40d4-bc66-05f1d779465b }, name: Name(\"nexus-0a835148-a2f0-40d4-bc66-05f1d779465b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 72])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2bea071a-6ee2-45ba-9a69-93a0e2e5b31b, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 8c296dc4-9b38-40c8-9549-80b2967f6cdc, kind: Service { id: 2bea071a-6ee2-45ba-9a69-93a0e2e5b31b }, name: Name(\"nexus-2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8dc5039d-cb88-49dd-8ab8-2ca47e481b18, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: e79fbd89-0714-4c5d-b550-de2def94225f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 16f9d651-64fc-4e3c-a69b-bda36e263e78, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: afeebd42-9418-4a17-adf1-4ce6f3cadd20, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 5eefa476-ecad-43d0-a297-e34591893cf5, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a18cc15b-758a-467a-8a3b-ed115f6e1135, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 36c43d98-b274-450d-98d0-db7e0ee7514c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 9227e547-c76f-4bc5-b6f2-101d008d915e, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 608a3190-2550-47a2-a23f-81797b571827, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: fca244c4-2682-42f7-9297-53c69ba4caa1, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:17.184468371Z","hostname":"buskin","pid":1661,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6507a4bd-9097-45d0-9787-1240feff9525, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:17.185157842Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6507a4bd-9097-45d0-9787-1240feff9525, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:17.185180483Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:17.20827593Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:17.208332792Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:17.229551403Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:17.229604115Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 416eb39a-1cea-4752-bc22-195679c44853, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:17.248891507Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 416eb39a-1cea-4752-bc22-195679c44853, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:17.248943269Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bdd869e5-e908-42b7-afb7-35517ea5ca46, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:17.267006444Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bdd869e5-e908-42b7-afb7-35517ea5ca46, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:17.267063666Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd70b00d-6277-40dc-9a7a-524663f1a340, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:17.286115212Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bd70b00d-6277-40dc-9a7a-524663f1a340, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:17.286182703Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:17.305530958Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:17.3055892Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 777ca252-0e2b-4952-8163-fd713097cfd8, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:17.325198002Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 777ca252-0e2b-4952-8163-fd713097cfd8, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:17.325251243Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:17.344244677Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:17.344298299Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:17.365613062Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:17.365678335Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:17.385426041Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:17.385474862Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:17.40560779Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:17.405665862Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5eefa476-ecad-43d0-a297-e34591893cf5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:17.423551112Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5eefa476-ecad-43d0-a297-e34591893cf5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:17.423601594Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a18cc15b-758a-467a-8a3b-ed115f6e1135, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:17.443054921Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a18cc15b-758a-467a-8a3b-ed115f6e1135, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:17.443104243Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 36c43d98-b274-450d-98d0-db7e0ee7514c, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:17.464125358Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 36c43d98-b274-450d-98d0-db7e0ee7514c, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:17.46417767Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9227e547-c76f-4bc5-b6f2-101d008d915e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:17.482932946Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9227e547-c76f-4bc5-b6f2-101d008d915e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:17.482995358Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:17.503696423Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:17.503754145Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 608a3190-2550-47a2-a23f-81797b571827, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:17.522402138Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 608a3190-2550-47a2-a23f-81797b571827, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:17.52245265Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:17.540776392Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:17.540828254Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fca244c4-2682-42f7-9297-53c69ba4caa1, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:17.55922529Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fca244c4-2682-42f7-9297-53c69ba4caa1, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:17.559285762Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:17.577680047Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:17.577746839Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:17.577973676Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:17.5781179Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:17.679337677Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:17.679398249Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:17.679405199Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:17.67941132Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:17.67941607Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:17.67942077Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:17.67944296Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:17.679447731Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:17.679451281Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:17.679456581Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:17.679460971Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:17.679466281Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:17.679470612Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:17.679475961Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:17.679484612Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:17.679490312Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:17.679494682Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:17.679499122Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:17.679503693Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:17.679620776Z","hostname":"buskin","pid":1661,"uri":"/omicron-zones","method":"PUT","req_id":"390c7673-de9e-4386-8512-dc427a265d02","remote_addr":"[fd00:1122:3344:101::1]:58669","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\", TimeNotSynchronized), (\"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4\", TimeNotSynchronized), (\"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e\", TimeNotSynchronized), (\"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135\", TimeNotSynchronized), (\"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f\", TimeNotSynchronized), (\"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71\", TimeNotSynchronized), (\"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\", TimeNotSynchronized), (\"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a\", TimeNotSynchronized), (\"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b\", TimeNotSynchronized), (\"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a\", TimeNotSynchronized), (\"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b\", TimeNotSynchronized), (\"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78\", TimeNotSynchronized), (\"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c\", TimeNotSynchronized), (\"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20\", TimeNotSynchronized), (\"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1\", TimeNotSynchronized), (\"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b\", TimeNotSynchronized), (\"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18\", TimeNotSynchronized), (\"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5\", TimeNotSynchronized), (\"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827\", TimeNotSynchronized)]","latency_us":494727,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:17.679851703Z","hostname":"buskin","pid":1661,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"182.09737ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"390c7673-de9e-4386-8512-dc427a265d02\", \"content-length\": \"124\", \"date\": \"Thu, 07 Mar 2024 22:45:17 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"390c7673-de9e-4386-8512-dc427a265d02\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 6507a4bd-9097-45d0-9787-1240feff9525, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 65847932-72ba-40f7-b62c-5adce2529400, kind: Service { id: f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf }, name: Name(\"ntp-f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 416eb39a-1cea-4752-bc22-195679c44853, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bdd869e5-e908-42b7-afb7-35517ea5ca46, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bd70b00d-6277-40dc-9a7a-524663f1a340, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 777ca252-0e2b-4952-8163-fd713097cfd8, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 0b3b3eae-a95f-4091-984a-93fccf167e79, kind: Service { id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c }, name: Name(\"external-dns-e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: ca9c5211-aa62-4394-a496-e20bf51739bc, kind: Service { id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b }, name: Name(\"external-dns-472c8e8c-ac8f-4193-921c-49dd3ceb288b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 761d7d83-4f44-4dad-ba8f-694aeae46e71, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 1eeeae19-992a-47f8-9e7b-dc93c05f0958, kind: Service { id: 761d7d83-4f44-4dad-ba8f-694aeae46e71 }, name: Name(\"nexus-761d7d83-4f44-4dad-ba8f-694aeae46e71\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a835148-a2f0-40d4-bc66-05f1d779465b, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 89eadaf9-099e-4929-ad88-e7ee2258e13d, kind: Service { id: 0a835148-a2f0-40d4-bc66-05f1d779465b }, name: Name(\"nexus-0a835148-a2f0-40d4-bc66-05f1d779465b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 72])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2bea071a-6ee2-45ba-9a69-93a0e2e5b31b, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 8c296dc4-9b38-40c8-9549-80b2967f6cdc, kind: Service { id: 2bea071a-6ee2-45ba-9a69-93a0e2e5b31b }, name: Name(\"nexus-2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8dc5039d-cb88-49dd-8ab8-2ca47e481b18, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: e79fbd89-0714-4c5d-b550-de2def94225f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 16f9d651-64fc-4e3c-a69b-bda36e263e78, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: afeebd42-9418-4a17-adf1-4ce6f3cadd20, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 5eefa476-ecad-43d0-a297-e34591893cf5, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a18cc15b-758a-467a-8a3b-ed115f6e1135, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 36c43d98-b274-450d-98d0-db7e0ee7514c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 9227e547-c76f-4bc5-b6f2-101d008d915e, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 608a3190-2550-47a2-a23f-81797b571827, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: fca244c4-2682-42f7-9297-53c69ba4caa1, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:17.862538461Z","hostname":"buskin","pid":1661,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6507a4bd-9097-45d0-9787-1240feff9525, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:17.863675925Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6507a4bd-9097-45d0-9787-1240feff9525, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:17.863690935Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:17.88603953Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:17.886084122Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:17.905170398Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:17.905209809Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 416eb39a-1cea-4752-bc22-195679c44853, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:17.923242184Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 416eb39a-1cea-4752-bc22-195679c44853, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:17.923296016Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bdd869e5-e908-42b7-afb7-35517ea5ca46, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:17.940962139Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bdd869e5-e908-42b7-afb7-35517ea5ca46, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:17.94100091Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd70b00d-6277-40dc-9a7a-524663f1a340, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:17.959641833Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bd70b00d-6277-40dc-9a7a-524663f1a340, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:17.959689125Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:17.977789291Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:17.977827703Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 777ca252-0e2b-4952-8163-fd713097cfd8, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:17.995648441Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 777ca252-0e2b-4952-8163-fd713097cfd8, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:17.995689542Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:18.013805909Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:18.013853111Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:18.032495674Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:18.032535355Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:18.050755565Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:18.050821257Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:18.069530972Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:18.069570634Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5eefa476-ecad-43d0-a297-e34591893cf5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:18.08767484Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5eefa476-ecad-43d0-a297-e34591893cf5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:18.087712491Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a18cc15b-758a-467a-8a3b-ed115f6e1135, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:18.105814398Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a18cc15b-758a-467a-8a3b-ed115f6e1135, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:18.105854209Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 36c43d98-b274-450d-98d0-db7e0ee7514c, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:18.124223744Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 36c43d98-b274-450d-98d0-db7e0ee7514c, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:18.124277986Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9227e547-c76f-4bc5-b6f2-101d008d915e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:18.142741284Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9227e547-c76f-4bc5-b6f2-101d008d915e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:18.142784285Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:18.160994885Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:18.161033966Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 608a3190-2550-47a2-a23f-81797b571827, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:18.179220535Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 608a3190-2550-47a2-a23f-81797b571827, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:18.179260436Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:18.197458766Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:18.197503627Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fca244c4-2682-42f7-9297-53c69ba4caa1, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:18.216070958Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fca244c4-2682-42f7-9297-53c69ba4caa1, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:18.21611504Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:18.234449203Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:18.234495925Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:18.234691101Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:18.234827715Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:18.333528716Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:18.333602578Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:18.333612888Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:18.333638789Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:18.333646479Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:18.33365471Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:18.33366312Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:18.33367178Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:18.33368029Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:18.33368742Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:18.333695521Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:18.333710931Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:18.333719561Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:18.333724502Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:18.333730902Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:18.333736122Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:18.333741242Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:18.333804754Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:18.333817635Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:18.333864816Z","hostname":"buskin","pid":1661,"uri":"/omicron-zones","method":"PUT","req_id":"55bb1712-2572-4c23-aad5-b016e2c1cd29","remote_addr":"[fd00:1122:3344:101::1]:58669","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c\", TimeNotSynchronized), (\"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827\", TimeNotSynchronized), (\"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b\", TimeNotSynchronized), (\"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20\", TimeNotSynchronized), (\"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e\", TimeNotSynchronized), (\"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b\", TimeNotSynchronized), (\"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a\", TimeNotSynchronized), (\"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\", TimeNotSynchronized), (\"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78\", TimeNotSynchronized), (\"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135\", TimeNotSynchronized), (\"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\", TimeNotSynchronized), (\"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71\", TimeNotSynchronized), (\"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5\", TimeNotSynchronized), (\"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1\", TimeNotSynchronized), (\"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18\", TimeNotSynchronized), (\"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b\", TimeNotSynchronized), (\"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4\", TimeNotSynchronized), (\"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a\", TimeNotSynchronized), (\"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f\", TimeNotSynchronized)]","latency_us":470862,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:18.334119484Z","hostname":"buskin","pid":1661,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"329.931898ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"55bb1712-2572-4c23-aad5-b016e2c1cd29\", \"content-length\": \"124\", \"date\": \"Thu, 07 Mar 2024 22:45:18 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"55bb1712-2572-4c23-aad5-b016e2c1cd29\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 6507a4bd-9097-45d0-9787-1240feff9525, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 65847932-72ba-40f7-b62c-5adce2529400, kind: Service { id: f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf }, name: Name(\"ntp-f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 416eb39a-1cea-4752-bc22-195679c44853, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bdd869e5-e908-42b7-afb7-35517ea5ca46, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bd70b00d-6277-40dc-9a7a-524663f1a340, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 777ca252-0e2b-4952-8163-fd713097cfd8, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 0b3b3eae-a95f-4091-984a-93fccf167e79, kind: Service { id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c }, name: Name(\"external-dns-e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: ca9c5211-aa62-4394-a496-e20bf51739bc, kind: Service { id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b }, name: Name(\"external-dns-472c8e8c-ac8f-4193-921c-49dd3ceb288b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 761d7d83-4f44-4dad-ba8f-694aeae46e71, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 1eeeae19-992a-47f8-9e7b-dc93c05f0958, kind: Service { id: 761d7d83-4f44-4dad-ba8f-694aeae46e71 }, name: Name(\"nexus-761d7d83-4f44-4dad-ba8f-694aeae46e71\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a835148-a2f0-40d4-bc66-05f1d779465b, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 89eadaf9-099e-4929-ad88-e7ee2258e13d, kind: Service { id: 0a835148-a2f0-40d4-bc66-05f1d779465b }, name: Name(\"nexus-0a835148-a2f0-40d4-bc66-05f1d779465b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 72])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2bea071a-6ee2-45ba-9a69-93a0e2e5b31b, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 8c296dc4-9b38-40c8-9549-80b2967f6cdc, kind: Service { id: 2bea071a-6ee2-45ba-9a69-93a0e2e5b31b }, name: Name(\"nexus-2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8dc5039d-cb88-49dd-8ab8-2ca47e481b18, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: e79fbd89-0714-4c5d-b550-de2def94225f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 16f9d651-64fc-4e3c-a69b-bda36e263e78, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: afeebd42-9418-4a17-adf1-4ce6f3cadd20, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 5eefa476-ecad-43d0-a297-e34591893cf5, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a18cc15b-758a-467a-8a3b-ed115f6e1135, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 36c43d98-b274-450d-98d0-db7e0ee7514c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 9227e547-c76f-4bc5-b6f2-101d008d915e, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 608a3190-2550-47a2-a23f-81797b571827, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: fca244c4-2682-42f7-9297-53c69ba4caa1, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:18.665045278Z","hostname":"buskin","pid":1661,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6507a4bd-9097-45d0-9787-1240feff9525, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:18.665948965Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6507a4bd-9097-45d0-9787-1240feff9525, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:18.665962575Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:18.687501416Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:18.689842987Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:18.705978464Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:18.706037506Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 416eb39a-1cea-4752-bc22-195679c44853, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:18.72438459Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 416eb39a-1cea-4752-bc22-195679c44853, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:18.724446042Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bdd869e5-e908-42b7-afb7-35517ea5ca46, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:18.744703064Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bdd869e5-e908-42b7-afb7-35517ea5ca46, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:18.744760465Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd70b00d-6277-40dc-9a7a-524663f1a340, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:18.763324736Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bd70b00d-6277-40dc-9a7a-524663f1a340, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:18.763386778Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:18.782129394Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:18.784664441Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 777ca252-0e2b-4952-8163-fd713097cfd8, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:18.800317783Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 777ca252-0e2b-4952-8163-fd713097cfd8, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:18.800373375Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:18.819472502Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:18.819530354Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:18.837856547Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:18.837916639Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:18.857586323Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:18.857643285Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:18.877560236Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:18.877614288Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5eefa476-ecad-43d0-a297-e34591893cf5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:18.896020314Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5eefa476-ecad-43d0-a297-e34591893cf5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:18.896075425Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a18cc15b-758a-467a-8a3b-ed115f6e1135, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:18.914361538Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a18cc15b-758a-467a-8a3b-ed115f6e1135, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:18.91442304Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 36c43d98-b274-450d-98d0-db7e0ee7514c, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:18.933163986Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 36c43d98-b274-450d-98d0-db7e0ee7514c, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:18.933221918Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9227e547-c76f-4bc5-b6f2-101d008d915e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:18.953635344Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9227e547-c76f-4bc5-b6f2-101d008d915e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:18.953698886Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:18.972336269Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:18.972396331Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 608a3190-2550-47a2-a23f-81797b571827, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:18.991108406Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 608a3190-2550-47a2-a23f-81797b571827, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:18.991173958Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:19.011273805Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:19.011332457Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fca244c4-2682-42f7-9297-53c69ba4caa1, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:19.031242388Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fca244c4-2682-42f7-9297-53c69ba4caa1, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:19.03130088Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:19.050805289Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:19.050894772Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:19.051118788Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:19.051266963Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:19.148977094Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:19.149031765Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:19.149039826Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:19.149048276Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:19.149054676Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:19.149059936Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:19.149064277Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:19.149070617Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:19.149081707Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:19.149088517Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:19.149094807Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:19.149101658Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:19.149107008Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:19.149113348Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:19.149118868Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:19.149135989Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:19.149141899Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:19.149146299Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:19.149151719Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:19.149234631Z","hostname":"buskin","pid":1661,"uri":"/omicron-zones","method":"PUT","req_id":"b9d39476-b3aa-4e98-9ec1-d613ecb88acb","remote_addr":"[fd00:1122:3344:101::1]:58669","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71\", TimeNotSynchronized), (\"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\", TimeNotSynchronized), (\"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c\", TimeNotSynchronized), (\"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f\", TimeNotSynchronized), (\"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a\", TimeNotSynchronized), (\"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1\", TimeNotSynchronized), (\"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b\", TimeNotSynchronized), (\"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b\", TimeNotSynchronized), (\"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18\", TimeNotSynchronized), (\"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20\", TimeNotSynchronized), (\"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4\", TimeNotSynchronized), (\"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e\", TimeNotSynchronized), (\"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827\", TimeNotSynchronized), (\"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a\", TimeNotSynchronized), (\"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78\", TimeNotSynchronized), (\"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\", TimeNotSynchronized), (\"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5\", TimeNotSynchronized), (\"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b\", TimeNotSynchronized), (\"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135\", TimeNotSynchronized)]","latency_us":483712,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:19.149485889Z","hostname":"buskin","pid":1661,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"205.62415ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b9d39476-b3aa-4e98-9ec1-d613ecb88acb\", \"content-length\": \"124\", \"date\": \"Thu, 07 Mar 2024 22:45:19 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b9d39476-b3aa-4e98-9ec1-d613ecb88acb\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 6507a4bd-9097-45d0-9787-1240feff9525, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 65847932-72ba-40f7-b62c-5adce2529400, kind: Service { id: f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf }, name: Name(\"ntp-f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 416eb39a-1cea-4752-bc22-195679c44853, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bdd869e5-e908-42b7-afb7-35517ea5ca46, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bd70b00d-6277-40dc-9a7a-524663f1a340, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 777ca252-0e2b-4952-8163-fd713097cfd8, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 0b3b3eae-a95f-4091-984a-93fccf167e79, kind: Service { id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c }, name: Name(\"external-dns-e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: ca9c5211-aa62-4394-a496-e20bf51739bc, kind: Service { id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b }, name: Name(\"external-dns-472c8e8c-ac8f-4193-921c-49dd3ceb288b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 761d7d83-4f44-4dad-ba8f-694aeae46e71, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 1eeeae19-992a-47f8-9e7b-dc93c05f0958, kind: Service { id: 761d7d83-4f44-4dad-ba8f-694aeae46e71 }, name: Name(\"nexus-761d7d83-4f44-4dad-ba8f-694aeae46e71\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a835148-a2f0-40d4-bc66-05f1d779465b, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 89eadaf9-099e-4929-ad88-e7ee2258e13d, kind: Service { id: 0a835148-a2f0-40d4-bc66-05f1d779465b }, name: Name(\"nexus-0a835148-a2f0-40d4-bc66-05f1d779465b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 72])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2bea071a-6ee2-45ba-9a69-93a0e2e5b31b, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 8c296dc4-9b38-40c8-9549-80b2967f6cdc, kind: Service { id: 2bea071a-6ee2-45ba-9a69-93a0e2e5b31b }, name: Name(\"nexus-2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8dc5039d-cb88-49dd-8ab8-2ca47e481b18, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: e79fbd89-0714-4c5d-b550-de2def94225f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 16f9d651-64fc-4e3c-a69b-bda36e263e78, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: afeebd42-9418-4a17-adf1-4ce6f3cadd20, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 5eefa476-ecad-43d0-a297-e34591893cf5, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a18cc15b-758a-467a-8a3b-ed115f6e1135, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 36c43d98-b274-450d-98d0-db7e0ee7514c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 9227e547-c76f-4bc5-b6f2-101d008d915e, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 608a3190-2550-47a2-a23f-81797b571827, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: fca244c4-2682-42f7-9297-53c69ba4caa1, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:19.35676439Z","hostname":"buskin","pid":1661,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6507a4bd-9097-45d0-9787-1240feff9525, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:19.357469331Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6507a4bd-9097-45d0-9787-1240feff9525, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:19.357484381Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:19.379402773Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:19.379471405Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:19.399040916Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:19.399094318Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 416eb39a-1cea-4752-bc22-195679c44853, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:19.419151494Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 416eb39a-1cea-4752-bc22-195679c44853, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:19.419212986Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bdd869e5-e908-42b7-afb7-35517ea5ca46, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:19.43721726Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bdd869e5-e908-42b7-afb7-35517ea5ca46, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:19.437281931Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd70b00d-6277-40dc-9a7a-524663f1a340, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:19.457805571Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bd70b00d-6277-40dc-9a7a-524663f1a340, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:19.457862253Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:19.47763158Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:19.477685232Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 777ca252-0e2b-4952-8163-fd713097cfd8, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:19.496646824Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 777ca252-0e2b-4952-8163-fd713097cfd8, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:19.496704956Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:19.515836674Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:19.515896726Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:19.535149387Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:19.535211179Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:19.553545002Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:19.553599844Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:19.572979219Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:19.573034201Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5eefa476-ecad-43d0-a297-e34591893cf5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:19.593036215Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5eefa476-ecad-43d0-a297-e34591893cf5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:19.593098137Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a18cc15b-758a-467a-8a3b-ed115f6e1135, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:19.611960397Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a18cc15b-758a-467a-8a3b-ed115f6e1135, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:19.612018738Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 36c43d98-b274-450d-98d0-db7e0ee7514c, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:19.631334182Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 36c43d98-b274-450d-98d0-db7e0ee7514c, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:19.631389633Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9227e547-c76f-4bc5-b6f2-101d008d915e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:19.649768998Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9227e547-c76f-4bc5-b6f2-101d008d915e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:19.64982536Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:19.667972418Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:19.66802535Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 608a3190-2550-47a2-a23f-81797b571827, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:19.686852378Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 608a3190-2550-47a2-a23f-81797b571827, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:19.68690726Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:19.707313636Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:19.707366288Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fca244c4-2682-42f7-9297-53c69ba4caa1, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:19.725941809Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fca244c4-2682-42f7-9297-53c69ba4caa1, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:19.726003481Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:19.744652504Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:19.744731736Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:19.744951003Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:19.745107098Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:19.843416677Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:19.843467058Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:19.843473539Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:19.843479449Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:19.843483829Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:19.843489419Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:19.843498319Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:19.84350408Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:19.84350872Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:19.84351325Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:19.84351776Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:19.84352224Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:19.843543171Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:19.843548911Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:19.843553871Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:19.843558141Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:19.843563641Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:19.843567902Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:19.843572322Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:19.843770528Z","hostname":"buskin","pid":1661,"uri":"/omicron-zones","method":"PUT","req_id":"d386375d-1679-44c5-9800-dabf796af539","remote_addr":"[fd00:1122:3344:101::1]:58669","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\", TimeNotSynchronized), (\"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\", TimeNotSynchronized), (\"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18\", TimeNotSynchronized), (\"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71\", TimeNotSynchronized), (\"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5\", TimeNotSynchronized), (\"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78\", TimeNotSynchronized), (\"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20\", TimeNotSynchronized), (\"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135\", TimeNotSynchronized), (\"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c\", TimeNotSynchronized), (\"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1\", TimeNotSynchronized), (\"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f\", TimeNotSynchronized), (\"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a\", TimeNotSynchronized), (\"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e\", TimeNotSynchronized), (\"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4\", TimeNotSynchronized), (\"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827\", TimeNotSynchronized), (\"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b\", TimeNotSynchronized), (\"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a\", TimeNotSynchronized), (\"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b\", TimeNotSynchronized), (\"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b\", TimeNotSynchronized)]","latency_us":486461,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:19.843998665Z","hostname":"buskin","pid":1661,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"201.062517ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d386375d-1679-44c5-9800-dabf796af539\", \"content-length\": \"124\", \"date\": \"Thu, 07 Mar 2024 22:45:19 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d386375d-1679-44c5-9800-dabf796af539\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 6507a4bd-9097-45d0-9787-1240feff9525, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 65847932-72ba-40f7-b62c-5adce2529400, kind: Service { id: f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf }, name: Name(\"ntp-f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 416eb39a-1cea-4752-bc22-195679c44853, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bdd869e5-e908-42b7-afb7-35517ea5ca46, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bd70b00d-6277-40dc-9a7a-524663f1a340, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 777ca252-0e2b-4952-8163-fd713097cfd8, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 0b3b3eae-a95f-4091-984a-93fccf167e79, kind: Service { id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c }, name: Name(\"external-dns-e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: ca9c5211-aa62-4394-a496-e20bf51739bc, kind: Service { id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b }, name: Name(\"external-dns-472c8e8c-ac8f-4193-921c-49dd3ceb288b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 761d7d83-4f44-4dad-ba8f-694aeae46e71, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 1eeeae19-992a-47f8-9e7b-dc93c05f0958, kind: Service { id: 761d7d83-4f44-4dad-ba8f-694aeae46e71 }, name: Name(\"nexus-761d7d83-4f44-4dad-ba8f-694aeae46e71\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a835148-a2f0-40d4-bc66-05f1d779465b, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 89eadaf9-099e-4929-ad88-e7ee2258e13d, kind: Service { id: 0a835148-a2f0-40d4-bc66-05f1d779465b }, name: Name(\"nexus-0a835148-a2f0-40d4-bc66-05f1d779465b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 72])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2bea071a-6ee2-45ba-9a69-93a0e2e5b31b, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 8c296dc4-9b38-40c8-9549-80b2967f6cdc, kind: Service { id: 2bea071a-6ee2-45ba-9a69-93a0e2e5b31b }, name: Name(\"nexus-2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8dc5039d-cb88-49dd-8ab8-2ca47e481b18, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: e79fbd89-0714-4c5d-b550-de2def94225f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 16f9d651-64fc-4e3c-a69b-bda36e263e78, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: afeebd42-9418-4a17-adf1-4ce6f3cadd20, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 5eefa476-ecad-43d0-a297-e34591893cf5, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a18cc15b-758a-467a-8a3b-ed115f6e1135, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 36c43d98-b274-450d-98d0-db7e0ee7514c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 9227e547-c76f-4bc5-b6f2-101d008d915e, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 608a3190-2550-47a2-a23f-81797b571827, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: fca244c4-2682-42f7-9297-53c69ba4caa1, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:20.045561842Z","hostname":"buskin","pid":1661,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6507a4bd-9097-45d0-9787-1240feff9525, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:20.046268624Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6507a4bd-9097-45d0-9787-1240feff9525, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:20.046281124Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:20.069047712Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:20.069119584Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:20.089219691Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:20.089276193Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 416eb39a-1cea-4752-bc22-195679c44853, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:20.110952277Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 416eb39a-1cea-4752-bc22-195679c44853, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:20.111008399Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bdd869e5-e908-42b7-afb7-35517ea5ca46, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:20.129546279Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bdd869e5-e908-42b7-afb7-35517ea5ca46, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:20.129603951Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd70b00d-6277-40dc-9a7a-524663f1a340, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:20.148680457Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bd70b00d-6277-40dc-9a7a-524663f1a340, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:20.148741529Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:20.167389342Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:20.167440103Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 777ca252-0e2b-4952-8163-fd713097cfd8, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:20.185725366Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 777ca252-0e2b-4952-8163-fd713097cfd8, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:20.185778267Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:20.203927885Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:20.203983157Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:20.222892378Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:20.22295536Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:20.242866252Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:20.242926023Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:20.261455463Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:20.261505575Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5eefa476-ecad-43d0-a297-e34591893cf5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:20.280690834Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5eefa476-ecad-43d0-a297-e34591893cf5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:20.280757966Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a18cc15b-758a-467a-8a3b-ed115f6e1135, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:20.299911774Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a18cc15b-758a-467a-8a3b-ed115f6e1135, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:20.299968496Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 36c43d98-b274-450d-98d0-db7e0ee7514c, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:20.319414053Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 36c43d98-b274-450d-98d0-db7e0ee7514c, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:20.322000791Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9227e547-c76f-4bc5-b6f2-101d008d915e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:20.338646774Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9227e547-c76f-4bc5-b6f2-101d008d915e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:20.338708236Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:20.357066861Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:20.357122132Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 608a3190-2550-47a2-a23f-81797b571827, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:20.376112516Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 608a3190-2550-47a2-a23f-81797b571827, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:20.37625531Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:20.394977056Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:20.395039048Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fca244c4-2682-42f7-9297-53c69ba4caa1, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:20.414460534Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fca244c4-2682-42f7-9297-53c69ba4caa1, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:20.414536356Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:20.433093257Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:20.433160369Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:20.433410716Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:20.433552271Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:20.529510109Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:20.52956506Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:20.529573341Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:20.529579941Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:20.529584691Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:20.529590421Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:20.529595261Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:20.529602652Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:20.529609112Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:20.529616272Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:20.529635753Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:20.529642133Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:20.529649343Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:20.529655233Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:20.529660934Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:20.529667994Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:20.529675054Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:20.529680674Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:20.529685134Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:20.529797887Z","hostname":"buskin","pid":1661,"uri":"/omicron-zones","method":"PUT","req_id":"75515865-b026-4f40-87b4-8f048e99bcf3","remote_addr":"[fd00:1122:3344:101::1]:58669","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20\", TimeNotSynchronized), (\"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135\", TimeNotSynchronized), (\"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827\", TimeNotSynchronized), (\"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b\", TimeNotSynchronized), (\"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\", TimeNotSynchronized), (\"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b\", TimeNotSynchronized), (\"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f\", TimeNotSynchronized), (\"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\", TimeNotSynchronized), (\"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c\", TimeNotSynchronized), (\"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5\", TimeNotSynchronized), (\"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a\", TimeNotSynchronized), (\"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78\", TimeNotSynchronized), (\"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1\", TimeNotSynchronized), (\"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a\", TimeNotSynchronized), (\"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4\", TimeNotSynchronized), (\"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e\", TimeNotSynchronized), (\"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b\", TimeNotSynchronized), (\"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71\", TimeNotSynchronized), (\"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18\", TimeNotSynchronized)]","latency_us":483754,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:20.530054465Z","hostname":"buskin","pid":1661,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"611.253302ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"75515865-b026-4f40-87b4-8f048e99bcf3\", \"content-length\": \"124\", \"date\": \"Thu, 07 Mar 2024 22:45:20 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"75515865-b026-4f40-87b4-8f048e99bcf3\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 6507a4bd-9097-45d0-9787-1240feff9525, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 65847932-72ba-40f7-b62c-5adce2529400, kind: Service { id: f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf }, name: Name(\"ntp-f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 416eb39a-1cea-4752-bc22-195679c44853, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bdd869e5-e908-42b7-afb7-35517ea5ca46, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bd70b00d-6277-40dc-9a7a-524663f1a340, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 777ca252-0e2b-4952-8163-fd713097cfd8, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 0b3b3eae-a95f-4091-984a-93fccf167e79, kind: Service { id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c }, name: Name(\"external-dns-e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: ca9c5211-aa62-4394-a496-e20bf51739bc, kind: Service { id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b }, name: Name(\"external-dns-472c8e8c-ac8f-4193-921c-49dd3ceb288b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 761d7d83-4f44-4dad-ba8f-694aeae46e71, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 1eeeae19-992a-47f8-9e7b-dc93c05f0958, kind: Service { id: 761d7d83-4f44-4dad-ba8f-694aeae46e71 }, name: Name(\"nexus-761d7d83-4f44-4dad-ba8f-694aeae46e71\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a835148-a2f0-40d4-bc66-05f1d779465b, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 89eadaf9-099e-4929-ad88-e7ee2258e13d, kind: Service { id: 0a835148-a2f0-40d4-bc66-05f1d779465b }, name: Name(\"nexus-0a835148-a2f0-40d4-bc66-05f1d779465b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 72])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2bea071a-6ee2-45ba-9a69-93a0e2e5b31b, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 8c296dc4-9b38-40c8-9549-80b2967f6cdc, kind: Service { id: 2bea071a-6ee2-45ba-9a69-93a0e2e5b31b }, name: Name(\"nexus-2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8dc5039d-cb88-49dd-8ab8-2ca47e481b18, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: e79fbd89-0714-4c5d-b550-de2def94225f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 16f9d651-64fc-4e3c-a69b-bda36e263e78, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: afeebd42-9418-4a17-adf1-4ce6f3cadd20, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 5eefa476-ecad-43d0-a297-e34591893cf5, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a18cc15b-758a-467a-8a3b-ed115f6e1135, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 36c43d98-b274-450d-98d0-db7e0ee7514c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 9227e547-c76f-4bc5-b6f2-101d008d915e, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 608a3190-2550-47a2-a23f-81797b571827, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: fca244c4-2682-42f7-9297-53c69ba4caa1, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:21.143259965Z","hostname":"buskin","pid":1661,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6507a4bd-9097-45d0-9787-1240feff9525, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:21.144534264Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6507a4bd-9097-45d0-9787-1240feff9525, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:21.144554344Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:21.166682572Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:21.166741584Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:21.188018897Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:21.188076308Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 416eb39a-1cea-4752-bc22-195679c44853, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:21.208197916Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 416eb39a-1cea-4752-bc22-195679c44853, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:21.208256058Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bdd869e5-e908-42b7-afb7-35517ea5ca46, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:21.228217011Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bdd869e5-e908-42b7-afb7-35517ea5ca46, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:21.230810039Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd70b00d-6277-40dc-9a7a-524663f1a340, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:21.247097821Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bd70b00d-6277-40dc-9a7a-524663f1a340, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:21.247156483Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:21.268670603Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:21.268719764Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 777ca252-0e2b-4952-8163-fd713097cfd8, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:21.28843799Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 777ca252-0e2b-4952-8163-fd713097cfd8, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:21.288476811Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:21.30863861Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:21.308676741Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:21.327600293Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:21.327644674Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:21.347909626Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:21.347952437Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:21.366666793Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:21.369104596Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5eefa476-ecad-43d0-a297-e34591893cf5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:21.385310865Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5eefa476-ecad-43d0-a297-e34591893cf5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:21.385355327Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a18cc15b-758a-467a-8a3b-ed115f6e1135, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:21.404567987Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a18cc15b-758a-467a-8a3b-ed115f6e1135, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:21.404605838Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 36c43d98-b274-450d-98d0-db7e0ee7514c, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:21.425058016Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 36c43d98-b274-450d-98d0-db7e0ee7514c, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:21.425123328Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9227e547-c76f-4bc5-b6f2-101d008d915e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:21.443564535Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9227e547-c76f-4bc5-b6f2-101d008d915e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:21.443616957Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:21.464217129Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:21.46425651Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 608a3190-2550-47a2-a23f-81797b571827, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:21.482991846Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 608a3190-2550-47a2-a23f-81797b571827, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:21.483029397Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:21.502720702Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:21.502779323Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fca244c4-2682-42f7-9297-53c69ba4caa1, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:21.522301383Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fca244c4-2682-42f7-9297-53c69ba4caa1, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:21.522338994Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:21.541531093Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:21.541588745Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:21.541771631Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:21.541983357Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:21.639276255Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:21.639335957Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:21.639343148Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:21.639350268Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:21.639356608Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:21.639361188Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:21.639367008Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:21.639383899Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:21.639390099Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:21.639395699Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:21.639401229Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:21.63940853Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:21.6394146Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:21.63942217Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:21.63942818Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:21.63943548Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:21.639447241Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:21.639453921Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:21.639461271Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:21.639573375Z","hostname":"buskin","pid":1661,"uri":"/omicron-zones","method":"PUT","req_id":"04574a74-93ce-4dd4-972e-6c8f35e0764f","remote_addr":"[fd00:1122:3344:101::1]:58669","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b\", TimeNotSynchronized), (\"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b\", TimeNotSynchronized), (\"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5\", TimeNotSynchronized), (\"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a\", TimeNotSynchronized), (\"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\", TimeNotSynchronized), (\"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1\", TimeNotSynchronized), (\"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e\", TimeNotSynchronized), (\"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78\", TimeNotSynchronized), (\"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4\", TimeNotSynchronized), (\"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135\", TimeNotSynchronized), (\"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c\", TimeNotSynchronized), (\"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827\", TimeNotSynchronized), (\"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71\", TimeNotSynchronized), (\"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b\", TimeNotSynchronized), (\"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20\", TimeNotSynchronized), (\"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18\", TimeNotSynchronized), (\"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a\", TimeNotSynchronized), (\"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\", TimeNotSynchronized), (\"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f\", TimeNotSynchronized)]","latency_us":495727,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:21.639859023Z","hostname":"buskin","pid":1661,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"376.675503ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"04574a74-93ce-4dd4-972e-6c8f35e0764f\", \"content-length\": \"124\", \"date\": \"Thu, 07 Mar 2024 22:45:21 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"04574a74-93ce-4dd4-972e-6c8f35e0764f\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 6507a4bd-9097-45d0-9787-1240feff9525, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 65847932-72ba-40f7-b62c-5adce2529400, kind: Service { id: f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf }, name: Name(\"ntp-f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 416eb39a-1cea-4752-bc22-195679c44853, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bdd869e5-e908-42b7-afb7-35517ea5ca46, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bd70b00d-6277-40dc-9a7a-524663f1a340, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 777ca252-0e2b-4952-8163-fd713097cfd8, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 0b3b3eae-a95f-4091-984a-93fccf167e79, kind: Service { id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c }, name: Name(\"external-dns-e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: ca9c5211-aa62-4394-a496-e20bf51739bc, kind: Service { id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b }, name: Name(\"external-dns-472c8e8c-ac8f-4193-921c-49dd3ceb288b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 761d7d83-4f44-4dad-ba8f-694aeae46e71, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 1eeeae19-992a-47f8-9e7b-dc93c05f0958, kind: Service { id: 761d7d83-4f44-4dad-ba8f-694aeae46e71 }, name: Name(\"nexus-761d7d83-4f44-4dad-ba8f-694aeae46e71\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a835148-a2f0-40d4-bc66-05f1d779465b, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 89eadaf9-099e-4929-ad88-e7ee2258e13d, kind: Service { id: 0a835148-a2f0-40d4-bc66-05f1d779465b }, name: Name(\"nexus-0a835148-a2f0-40d4-bc66-05f1d779465b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 72])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2bea071a-6ee2-45ba-9a69-93a0e2e5b31b, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 8c296dc4-9b38-40c8-9549-80b2967f6cdc, kind: Service { id: 2bea071a-6ee2-45ba-9a69-93a0e2e5b31b }, name: Name(\"nexus-2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8dc5039d-cb88-49dd-8ab8-2ca47e481b18, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: e79fbd89-0714-4c5d-b550-de2def94225f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 16f9d651-64fc-4e3c-a69b-bda36e263e78, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: afeebd42-9418-4a17-adf1-4ce6f3cadd20, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 5eefa476-ecad-43d0-a297-e34591893cf5, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a18cc15b-758a-467a-8a3b-ed115f6e1135, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 36c43d98-b274-450d-98d0-db7e0ee7514c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 9227e547-c76f-4bc5-b6f2-101d008d915e, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 608a3190-2550-47a2-a23f-81797b571827, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: fca244c4-2682-42f7-9297-53c69ba4caa1, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:22.017412106Z","hostname":"buskin","pid":1661,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6507a4bd-9097-45d0-9787-1240feff9525, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:22.018104517Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6507a4bd-9097-45d0-9787-1240feff9525, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:22.018124058Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:22.041472333Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:22.041530865Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:22.061109586Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:22.061169278Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 416eb39a-1cea-4752-bc22-195679c44853, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:22.079328906Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 416eb39a-1cea-4752-bc22-195679c44853, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:22.079395498Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bdd869e5-e908-42b7-afb7-35517ea5ca46, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:22.09734251Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bdd869e5-e908-42b7-afb7-35517ea5ca46, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:22.097392972Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd70b00d-6277-40dc-9a7a-524663f1a340, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:22.115029835Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bd70b00d-6277-40dc-9a7a-524663f1a340, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:22.115085276Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:22.135242955Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:22.135298687Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 777ca252-0e2b-4952-8163-fd713097cfd8, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:22.154333322Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 777ca252-0e2b-4952-8163-fd713097cfd8, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:22.154397964Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:22.173290684Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:22.173347556Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:22.191579287Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:22.191628138Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:22.211828348Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:22.21188196Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:22.232458671Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:22.232509483Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5eefa476-ecad-43d0-a297-e34591893cf5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:22.250940129Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5eefa476-ecad-43d0-a297-e34591893cf5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:22.250990561Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a18cc15b-758a-467a-8a3b-ed115f6e1135, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:22.271004365Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a18cc15b-758a-467a-8a3b-ed115f6e1135, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:22.271055947Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 36c43d98-b274-450d-98d0-db7e0ee7514c, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:22.290924007Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 36c43d98-b274-450d-98d0-db7e0ee7514c, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:22.290993119Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9227e547-c76f-4bc5-b6f2-101d008d915e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:22.312734806Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9227e547-c76f-4bc5-b6f2-101d008d915e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:22.312792848Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:22.332195483Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:22.332245585Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 608a3190-2550-47a2-a23f-81797b571827, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:22.351553998Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 608a3190-2550-47a2-a23f-81797b571827, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:22.35161108Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:22.37083355Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:22.370900463Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fca244c4-2682-42f7-9297-53c69ba4caa1, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:22.389657349Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fca244c4-2682-42f7-9297-53c69ba4caa1, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:22.389708951Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:22.409846989Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:22.409913081Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:22.410112717Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:22.410265701Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:22.507534739Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:22.507611972Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:22.507623512Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:22.507630782Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:22.507651113Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:22.507658513Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:22.507665453Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:22.507670733Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:22.507677323Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:22.507682594Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:22.507687654Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:22.507692914Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:22.507698174Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:22.507703184Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:22.507710015Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:22.507721145Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:22.507727965Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:22.507734875Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:22.507741405Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:22.507783347Z","hostname":"buskin","pid":1661,"uri":"/omicron-zones","method":"PUT","req_id":"27579742-ebeb-474d-8148-097401f5671c","remote_addr":"[fd00:1122:3344:101::1]:58669","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\", TimeNotSynchronized), (\"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f\", TimeNotSynchronized), (\"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20\", TimeNotSynchronized), (\"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71\", TimeNotSynchronized), (\"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4\", TimeNotSynchronized), (\"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b\", TimeNotSynchronized), (\"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18\", TimeNotSynchronized), (\"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b\", TimeNotSynchronized), (\"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a\", TimeNotSynchronized), (\"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78\", TimeNotSynchronized), (\"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5\", TimeNotSynchronized), (\"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135\", TimeNotSynchronized), (\"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b\", TimeNotSynchronized), (\"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a\", TimeNotSynchronized), (\"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e\", TimeNotSynchronized), (\"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827\", TimeNotSynchronized), (\"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c\", TimeNotSynchronized), (\"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1\", TimeNotSynchronized), (\"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\", TimeNotSynchronized)]","latency_us":489912,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:22.508052555Z","hostname":"buskin","pid":1661,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"642.075936ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"27579742-ebeb-474d-8148-097401f5671c\", \"content-length\": \"124\", \"date\": \"Thu, 07 Mar 2024 22:45:21 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"27579742-ebeb-474d-8148-097401f5671c\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 6507a4bd-9097-45d0-9787-1240feff9525, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 65847932-72ba-40f7-b62c-5adce2529400, kind: Service { id: f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf }, name: Name(\"ntp-f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 416eb39a-1cea-4752-bc22-195679c44853, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bdd869e5-e908-42b7-afb7-35517ea5ca46, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bd70b00d-6277-40dc-9a7a-524663f1a340, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 777ca252-0e2b-4952-8163-fd713097cfd8, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 0b3b3eae-a95f-4091-984a-93fccf167e79, kind: Service { id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c }, name: Name(\"external-dns-e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: ca9c5211-aa62-4394-a496-e20bf51739bc, kind: Service { id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b }, name: Name(\"external-dns-472c8e8c-ac8f-4193-921c-49dd3ceb288b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 761d7d83-4f44-4dad-ba8f-694aeae46e71, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 1eeeae19-992a-47f8-9e7b-dc93c05f0958, kind: Service { id: 761d7d83-4f44-4dad-ba8f-694aeae46e71 }, name: Name(\"nexus-761d7d83-4f44-4dad-ba8f-694aeae46e71\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a835148-a2f0-40d4-bc66-05f1d779465b, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 89eadaf9-099e-4929-ad88-e7ee2258e13d, kind: Service { id: 0a835148-a2f0-40d4-bc66-05f1d779465b }, name: Name(\"nexus-0a835148-a2f0-40d4-bc66-05f1d779465b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 72])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2bea071a-6ee2-45ba-9a69-93a0e2e5b31b, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 8c296dc4-9b38-40c8-9549-80b2967f6cdc, kind: Service { id: 2bea071a-6ee2-45ba-9a69-93a0e2e5b31b }, name: Name(\"nexus-2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8dc5039d-cb88-49dd-8ab8-2ca47e481b18, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: e79fbd89-0714-4c5d-b550-de2def94225f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 16f9d651-64fc-4e3c-a69b-bda36e263e78, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: afeebd42-9418-4a17-adf1-4ce6f3cadd20, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 5eefa476-ecad-43d0-a297-e34591893cf5, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a18cc15b-758a-467a-8a3b-ed115f6e1135, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 36c43d98-b274-450d-98d0-db7e0ee7514c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 9227e547-c76f-4bc5-b6f2-101d008d915e, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 608a3190-2550-47a2-a23f-81797b571827, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: fca244c4-2682-42f7-9297-53c69ba4caa1, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:23.151767176Z","hostname":"buskin","pid":1661,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6507a4bd-9097-45d0-9787-1240feff9525, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:23.152637062Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6507a4bd-9097-45d0-9787-1240feff9525, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:23.152650753Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:23.174345728Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:23.174394659Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:23.194456135Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:23.194503167Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 416eb39a-1cea-4752-bc22-195679c44853, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:23.213388867Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 416eb39a-1cea-4752-bc22-195679c44853, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:23.213445459Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bdd869e5-e908-42b7-afb7-35517ea5ca46, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:23.233319309Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bdd869e5-e908-42b7-afb7-35517ea5ca46, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:23.233378321Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd70b00d-6277-40dc-9a7a-524663f1a340, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:23.251710165Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bd70b00d-6277-40dc-9a7a-524663f1a340, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:23.251750146Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:23.270332647Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:23.270382938Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 777ca252-0e2b-4952-8163-fd713097cfd8, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:23.289662271Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 777ca252-0e2b-4952-8163-fd713097cfd8, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:23.289704112Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:23.310238472Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:23.310279333Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:23.330810214Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:23.330850665Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:23.351085146Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:23.351145388Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:23.370813152Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:23.370852343Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5eefa476-ecad-43d0-a297-e34591893cf5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:23.389421744Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5eefa476-ecad-43d0-a297-e34591893cf5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:23.389487046Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a18cc15b-758a-467a-8a3b-ed115f6e1135, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:23.40785417Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a18cc15b-758a-467a-8a3b-ed115f6e1135, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:23.407910332Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 36c43d98-b274-450d-98d0-db7e0ee7514c, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:23.426596667Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 36c43d98-b274-450d-98d0-db7e0ee7514c, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:23.426637448Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9227e547-c76f-4bc5-b6f2-101d008d915e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:23.445195998Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9227e547-c76f-4bc5-b6f2-101d008d915e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:23.445233539Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:23.465033557Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:23.465073058Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 608a3190-2550-47a2-a23f-81797b571827, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:23.482926078Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 608a3190-2550-47a2-a23f-81797b571827, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:23.4830005Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:23.502660634Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:23.502705025Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fca244c4-2682-42f7-9297-53c69ba4caa1, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:23.520904975Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fca244c4-2682-42f7-9297-53c69ba4caa1, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:23.520945856Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:23.540478255Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:23.540522667Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:23.540726083Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:23.540860027Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:23.638334341Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:23.638411593Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:23.638419324Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:23.638424274Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:23.638430074Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:23.638436564Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:23.638442304Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:23.638454425Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:23.638460435Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:23.638466075Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:23.638473785Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:23.638481126Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:23.638486996Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:23.638493336Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:23.638498936Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:23.638506076Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:23.638526747Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:23.638531327Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:23.638537087Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:23.63862012Z","hostname":"buskin","pid":1661,"uri":"/omicron-zones","method":"PUT","req_id":"25b59b91-72fd-4135-8253-e9f01fdc6da7","remote_addr":"[fd00:1122:3344:101::1]:58669","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b\", TimeNotSynchronized), (\"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1\", TimeNotSynchronized), (\"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b\", TimeNotSynchronized), (\"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71\", TimeNotSynchronized), (\"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c\", TimeNotSynchronized), (\"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827\", TimeNotSynchronized), (\"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5\", TimeNotSynchronized), (\"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a\", TimeNotSynchronized), (\"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b\", TimeNotSynchronized), (\"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\", TimeNotSynchronized), (\"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\", TimeNotSynchronized), (\"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a\", TimeNotSynchronized), (\"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e\", TimeNotSynchronized), (\"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f\", TimeNotSynchronized), (\"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18\", TimeNotSynchronized), (\"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78\", TimeNotSynchronized), (\"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20\", TimeNotSynchronized), (\"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4\", TimeNotSynchronized), (\"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135\", TimeNotSynchronized)]","latency_us":486242,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:23.638873397Z","hostname":"buskin","pid":1661,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"497.462613ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"25b59b91-72fd-4135-8253-e9f01fdc6da7\", \"content-length\": \"124\", \"date\": \"Thu, 07 Mar 2024 22:45:23 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"25b59b91-72fd-4135-8253-e9f01fdc6da7\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 6507a4bd-9097-45d0-9787-1240feff9525, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 65847932-72ba-40f7-b62c-5adce2529400, kind: Service { id: f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf }, name: Name(\"ntp-f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 416eb39a-1cea-4752-bc22-195679c44853, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bdd869e5-e908-42b7-afb7-35517ea5ca46, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bd70b00d-6277-40dc-9a7a-524663f1a340, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 777ca252-0e2b-4952-8163-fd713097cfd8, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 0b3b3eae-a95f-4091-984a-93fccf167e79, kind: Service { id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c }, name: Name(\"external-dns-e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: ca9c5211-aa62-4394-a496-e20bf51739bc, kind: Service { id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b }, name: Name(\"external-dns-472c8e8c-ac8f-4193-921c-49dd3ceb288b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 761d7d83-4f44-4dad-ba8f-694aeae46e71, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 1eeeae19-992a-47f8-9e7b-dc93c05f0958, kind: Service { id: 761d7d83-4f44-4dad-ba8f-694aeae46e71 }, name: Name(\"nexus-761d7d83-4f44-4dad-ba8f-694aeae46e71\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a835148-a2f0-40d4-bc66-05f1d779465b, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 89eadaf9-099e-4929-ad88-e7ee2258e13d, kind: Service { id: 0a835148-a2f0-40d4-bc66-05f1d779465b }, name: Name(\"nexus-0a835148-a2f0-40d4-bc66-05f1d779465b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 72])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2bea071a-6ee2-45ba-9a69-93a0e2e5b31b, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 8c296dc4-9b38-40c8-9549-80b2967f6cdc, kind: Service { id: 2bea071a-6ee2-45ba-9a69-93a0e2e5b31b }, name: Name(\"nexus-2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8dc5039d-cb88-49dd-8ab8-2ca47e481b18, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: e79fbd89-0714-4c5d-b550-de2def94225f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 16f9d651-64fc-4e3c-a69b-bda36e263e78, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: afeebd42-9418-4a17-adf1-4ce6f3cadd20, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 5eefa476-ecad-43d0-a297-e34591893cf5, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a18cc15b-758a-467a-8a3b-ed115f6e1135, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 36c43d98-b274-450d-98d0-db7e0ee7514c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 9227e547-c76f-4bc5-b6f2-101d008d915e, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 608a3190-2550-47a2-a23f-81797b571827, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: fca244c4-2682-42f7-9297-53c69ba4caa1, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:24.137325612Z","hostname":"buskin","pid":1661,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6507a4bd-9097-45d0-9787-1240feff9525, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:24.138011083Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6507a4bd-9097-45d0-9787-1240feff9525, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:24.138027903Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:24.1614357Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:24.161502942Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:24.182446215Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:24.185054283Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 416eb39a-1cea-4752-bc22-195679c44853, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:24.200905722Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 416eb39a-1cea-4752-bc22-195679c44853, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:24.200965994Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bdd869e5-e908-42b7-afb7-35517ea5ca46, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:24.221010359Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bdd869e5-e908-42b7-afb7-35517ea5ca46, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:24.221071261Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd70b00d-6277-40dc-9a7a-524663f1a340, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:24.240347633Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bd70b00d-6277-40dc-9a7a-524663f1a340, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:24.240403175Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:24.259206463Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:24.259263405Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 777ca252-0e2b-4952-8163-fd713097cfd8, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:24.277882847Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 777ca252-0e2b-4952-8163-fd713097cfd8, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:24.277935059Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:24.296725666Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:24.296775458Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:24.316402681Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:24.318534495Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:24.335503637Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:24.335554819Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:24.354622775Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:24.354679097Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5eefa476-ecad-43d0-a297-e34591893cf5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:24.372977139Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5eefa476-ecad-43d0-a297-e34591893cf5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:24.373050141Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a18cc15b-758a-467a-8a3b-ed115f6e1135, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:24.392166049Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a18cc15b-758a-467a-8a3b-ed115f6e1135, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:24.392226091Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 36c43d98-b274-450d-98d0-db7e0ee7514c, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:24.410560304Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 36c43d98-b274-450d-98d0-db7e0ee7514c, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:24.410614436Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9227e547-c76f-4bc5-b6f2-101d008d915e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:24.430548898Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9227e547-c76f-4bc5-b6f2-101d008d915e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:24.43060336Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:24.450303825Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:24.450374897Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 608a3190-2550-47a2-a23f-81797b571827, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:24.470369471Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 608a3190-2550-47a2-a23f-81797b571827, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:24.470428522Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:24.489385255Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:24.489436696Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fca244c4-2682-42f7-9297-53c69ba4caa1, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:24.508863753Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fca244c4-2682-42f7-9297-53c69ba4caa1, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:24.508920565Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:24.527585269Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:24.527667381Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:24.527878828Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:24.528019792Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:24.62664648Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:24.626696772Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:24.626710892Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:24.626716892Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:24.626721682Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:24.626726193Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:24.626732243Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:24.626736543Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:24.626743233Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:24.626748453Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:24.626753063Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:24.626758303Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:24.626762543Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:24.626783744Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:24.626788614Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:24.626794135Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:24.626798455Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:24.626802845Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:24.626806265Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:24.626886557Z","hostname":"buskin","pid":1661,"uri":"/omicron-zones","method":"PUT","req_id":"da4b5fb4-45ca-471b-88fd-c1a99d5374f2","remote_addr":"[fd00:1122:3344:101::1]:58669","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4\", TimeNotSynchronized), (\"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b\", TimeNotSynchronized), (\"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78\", TimeNotSynchronized), (\"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e\", TimeNotSynchronized), (\"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\", TimeNotSynchronized), (\"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f\", TimeNotSynchronized), (\"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135\", TimeNotSynchronized), (\"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18\", TimeNotSynchronized), (\"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5\", TimeNotSynchronized), (\"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c\", TimeNotSynchronized), (\"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b\", TimeNotSynchronized), (\"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827\", TimeNotSynchronized), (\"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71\", TimeNotSynchronized), (\"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a\", TimeNotSynchronized), (\"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\", TimeNotSynchronized), (\"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a\", TimeNotSynchronized), (\"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b\", TimeNotSynchronized), (\"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1\", TimeNotSynchronized), (\"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20\", TimeNotSynchronized)]","latency_us":489113,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:24.627083293Z","hostname":"buskin","pid":1661,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"672.405072ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"da4b5fb4-45ca-471b-88fd-c1a99d5374f2\", \"content-length\": \"124\", \"date\": \"Thu, 07 Mar 2024 22:45:24 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"da4b5fb4-45ca-471b-88fd-c1a99d5374f2\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 6507a4bd-9097-45d0-9787-1240feff9525, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 65847932-72ba-40f7-b62c-5adce2529400, kind: Service { id: f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf }, name: Name(\"ntp-f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 416eb39a-1cea-4752-bc22-195679c44853, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bdd869e5-e908-42b7-afb7-35517ea5ca46, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bd70b00d-6277-40dc-9a7a-524663f1a340, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 777ca252-0e2b-4952-8163-fd713097cfd8, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 0b3b3eae-a95f-4091-984a-93fccf167e79, kind: Service { id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c }, name: Name(\"external-dns-e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: ca9c5211-aa62-4394-a496-e20bf51739bc, kind: Service { id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b }, name: Name(\"external-dns-472c8e8c-ac8f-4193-921c-49dd3ceb288b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 761d7d83-4f44-4dad-ba8f-694aeae46e71, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 1eeeae19-992a-47f8-9e7b-dc93c05f0958, kind: Service { id: 761d7d83-4f44-4dad-ba8f-694aeae46e71 }, name: Name(\"nexus-761d7d83-4f44-4dad-ba8f-694aeae46e71\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a835148-a2f0-40d4-bc66-05f1d779465b, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 89eadaf9-099e-4929-ad88-e7ee2258e13d, kind: Service { id: 0a835148-a2f0-40d4-bc66-05f1d779465b }, name: Name(\"nexus-0a835148-a2f0-40d4-bc66-05f1d779465b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 72])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2bea071a-6ee2-45ba-9a69-93a0e2e5b31b, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 8c296dc4-9b38-40c8-9549-80b2967f6cdc, kind: Service { id: 2bea071a-6ee2-45ba-9a69-93a0e2e5b31b }, name: Name(\"nexus-2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8dc5039d-cb88-49dd-8ab8-2ca47e481b18, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: e79fbd89-0714-4c5d-b550-de2def94225f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 16f9d651-64fc-4e3c-a69b-bda36e263e78, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: afeebd42-9418-4a17-adf1-4ce6f3cadd20, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 5eefa476-ecad-43d0-a297-e34591893cf5, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a18cc15b-758a-467a-8a3b-ed115f6e1135, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 36c43d98-b274-450d-98d0-db7e0ee7514c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 9227e547-c76f-4bc5-b6f2-101d008d915e, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 608a3190-2550-47a2-a23f-81797b571827, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: fca244c4-2682-42f7-9297-53c69ba4caa1, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:25.300859873Z","hostname":"buskin","pid":1661,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6507a4bd-9097-45d0-9787-1240feff9525, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:25.301734909Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6507a4bd-9097-45d0-9787-1240feff9525, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:25.30174857Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:25.324776695Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:25.324849087Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:25.345632015Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:25.345697587Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 416eb39a-1cea-4752-bc22-195679c44853, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:25.369418734Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 416eb39a-1cea-4752-bc22-195679c44853, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:25.369478206Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bdd869e5-e908-42b7-afb7-35517ea5ca46, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:25.389299114Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bdd869e5-e908-42b7-afb7-35517ea5ca46, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:25.389356346Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd70b00d-6277-40dc-9a7a-524663f1a340, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:25.409217236Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bd70b00d-6277-40dc-9a7a-524663f1a340, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:25.409274968Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:25.429736866Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:25.429796737Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 777ca252-0e2b-4952-8163-fd713097cfd8, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:25.448383069Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 777ca252-0e2b-4952-8163-fd713097cfd8, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:25.448443271Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:25.467768264Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:25.467828846Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:25.487922003Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:25.487979834Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:25.506646108Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:25.50670149Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:25.526384454Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:25.526445356Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5eefa476-ecad-43d0-a297-e34591893cf5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:25.545215393Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5eefa476-ecad-43d0-a297-e34591893cf5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:25.545280564Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a18cc15b-758a-467a-8a3b-ed115f6e1135, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:25.565937018Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a18cc15b-758a-467a-8a3b-ed115f6e1135, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:25.56599561Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 36c43d98-b274-450d-98d0-db7e0ee7514c, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:25.587921092Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 36c43d98-b274-450d-98d0-db7e0ee7514c, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:25.587982234Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9227e547-c76f-4bc5-b6f2-101d008d915e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:25.607309948Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9227e547-c76f-4bc5-b6f2-101d008d915e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:25.60736299Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:25.626164687Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:25.62623998Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 608a3190-2550-47a2-a23f-81797b571827, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:25.64578349Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 608a3190-2550-47a2-a23f-81797b571827, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:25.645923964Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:25.665125154Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:25.665178656Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fca244c4-2682-42f7-9297-53c69ba4caa1, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:25.68386536Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fca244c4-2682-42f7-9297-53c69ba4caa1, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:25.683918382Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:25.70305781Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:25.703118822Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:25.703321468Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:25.703463292Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:25.802280606Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:25.802367579Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:25.80238164Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:25.80239021Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:25.80239725Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:25.80240343Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:25.802411321Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:25.802418921Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:25.802426921Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:25.802436771Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:25.802460942Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:25.802705009Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:25.80271315Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:25.80271732Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:25.80272167Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:25.80272582Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:25.80272996Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:25.80273702Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:25.80274131Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:25.802745711Z","hostname":"buskin","pid":1661,"uri":"/omicron-zones","method":"PUT","req_id":"58887699-1845-49bc-97b9-b071d8d3fc47","remote_addr":"[fd00:1122:3344:101::1]:58669","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\", TimeNotSynchronized), (\"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78\", TimeNotSynchronized), (\"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4\", TimeNotSynchronized), (\"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71\", TimeNotSynchronized), (\"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a\", TimeNotSynchronized), (\"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5\", TimeNotSynchronized), (\"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b\", TimeNotSynchronized), (\"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f\", TimeNotSynchronized), (\"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c\", TimeNotSynchronized), (\"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827\", TimeNotSynchronized), (\"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e\", TimeNotSynchronized), (\"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135\", TimeNotSynchronized), (\"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20\", TimeNotSynchronized), (\"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1\", TimeNotSynchronized), (\"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a\", TimeNotSynchronized), (\"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b\", TimeNotSynchronized), (\"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b\", TimeNotSynchronized), (\"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\", TimeNotSynchronized), (\"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18\", TimeNotSynchronized)]","latency_us":500998,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:25.802785922Z","hostname":"buskin","pid":1661,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"787.292794ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"58887699-1845-49bc-97b9-b071d8d3fc47\", \"content-length\": \"124\", \"date\": \"Thu, 07 Mar 2024 22:45:25 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"58887699-1845-49bc-97b9-b071d8d3fc47\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 6507a4bd-9097-45d0-9787-1240feff9525, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 65847932-72ba-40f7-b62c-5adce2529400, kind: Service { id: f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf }, name: Name(\"ntp-f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 416eb39a-1cea-4752-bc22-195679c44853, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bdd869e5-e908-42b7-afb7-35517ea5ca46, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bd70b00d-6277-40dc-9a7a-524663f1a340, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 777ca252-0e2b-4952-8163-fd713097cfd8, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 0b3b3eae-a95f-4091-984a-93fccf167e79, kind: Service { id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c }, name: Name(\"external-dns-e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: ca9c5211-aa62-4394-a496-e20bf51739bc, kind: Service { id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b }, name: Name(\"external-dns-472c8e8c-ac8f-4193-921c-49dd3ceb288b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 761d7d83-4f44-4dad-ba8f-694aeae46e71, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 1eeeae19-992a-47f8-9e7b-dc93c05f0958, kind: Service { id: 761d7d83-4f44-4dad-ba8f-694aeae46e71 }, name: Name(\"nexus-761d7d83-4f44-4dad-ba8f-694aeae46e71\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a835148-a2f0-40d4-bc66-05f1d779465b, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 89eadaf9-099e-4929-ad88-e7ee2258e13d, kind: Service { id: 0a835148-a2f0-40d4-bc66-05f1d779465b }, name: Name(\"nexus-0a835148-a2f0-40d4-bc66-05f1d779465b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 72])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2bea071a-6ee2-45ba-9a69-93a0e2e5b31b, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 8c296dc4-9b38-40c8-9549-80b2967f6cdc, kind: Service { id: 2bea071a-6ee2-45ba-9a69-93a0e2e5b31b }, name: Name(\"nexus-2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8dc5039d-cb88-49dd-8ab8-2ca47e481b18, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: e79fbd89-0714-4c5d-b550-de2def94225f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 16f9d651-64fc-4e3c-a69b-bda36e263e78, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: afeebd42-9418-4a17-adf1-4ce6f3cadd20, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 5eefa476-ecad-43d0-a297-e34591893cf5, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a18cc15b-758a-467a-8a3b-ed115f6e1135, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 36c43d98-b274-450d-98d0-db7e0ee7514c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 9227e547-c76f-4bc5-b6f2-101d008d915e, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 608a3190-2550-47a2-a23f-81797b571827, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: fca244c4-2682-42f7-9297-53c69ba4caa1, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:26.590669748Z","hostname":"buskin","pid":1661,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6507a4bd-9097-45d0-9787-1240feff9525, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:26.59140319Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6507a4bd-9097-45d0-9787-1240feff9525, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:26.59142049Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:26.614409154Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:26.614506667Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:26.636794961Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:26.636853302Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 416eb39a-1cea-4752-bc22-195679c44853, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:26.656332401Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 416eb39a-1cea-4752-bc22-195679c44853, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:26.656391442Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bdd869e5-e908-42b7-afb7-35517ea5ca46, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:26.677621854Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bdd869e5-e908-42b7-afb7-35517ea5ca46, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:26.677685485Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd70b00d-6277-40dc-9a7a-524663f1a340, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:26.696625538Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bd70b00d-6277-40dc-9a7a-524663f1a340, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:26.696680279Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:26.714790486Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:26.714844768Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 777ca252-0e2b-4952-8163-fd713097cfd8, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:26.734688757Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 777ca252-0e2b-4952-8163-fd713097cfd8, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:26.734747359Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:26.75532244Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:26.755378962Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:26.775322385Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:26.775381976Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:26.795237696Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:26.795301368Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:26.814355143Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:26.814414815Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5eefa476-ecad-43d0-a297-e34591893cf5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:26.833608865Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5eefa476-ecad-43d0-a297-e34591893cf5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:26.833668177Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a18cc15b-758a-467a-8a3b-ed115f6e1135, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:26.852912288Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a18cc15b-758a-467a-8a3b-ed115f6e1135, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:26.85296577Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 36c43d98-b274-450d-98d0-db7e0ee7514c, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:26.87250046Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 36c43d98-b274-450d-98d0-db7e0ee7514c, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:26.872554961Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9227e547-c76f-4bc5-b6f2-101d008d915e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:26.891526814Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9227e547-c76f-4bc5-b6f2-101d008d915e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:26.891592606Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:26.910963391Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:26.911017863Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 608a3190-2550-47a2-a23f-81797b571827, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:26.929074648Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 608a3190-2550-47a2-a23f-81797b571827, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:26.92912961Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:26.948310409Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:26.948365061Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fca244c4-2682-42f7-9297-53c69ba4caa1, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:26.967260351Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fca244c4-2682-42f7-9297-53c69ba4caa1, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:26.967320653Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:26.986859434Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:26.986927356Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:26.987139472Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:26.987303517Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:27.086286806Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:27.086338228Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:27.086345208Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:27.086350828Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:27.086356348Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:27.086360009Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:27.086364519Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:27.086385189Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:27.086391089Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:27.08639545Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:27.08640009Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:27.08640541Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:27.08641006Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:27.08641459Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:27.08642005Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:27.086429261Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:27.086434781Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:27.086439171Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:27.086444621Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:27.086537514Z","hostname":"buskin","pid":1661,"uri":"/omicron-zones","method":"PUT","req_id":"1d38f008-f81c-4514-bee1-6a463616f44b","remote_addr":"[fd00:1122:3344:101::1]:58669","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827\", TimeNotSynchronized), (\"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4\", TimeNotSynchronized), (\"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b\", TimeNotSynchronized), (\"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135\", TimeNotSynchronized), (\"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e\", TimeNotSynchronized), (\"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\", TimeNotSynchronized), (\"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20\", TimeNotSynchronized), (\"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1\", TimeNotSynchronized), (\"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18\", TimeNotSynchronized), (\"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5\", TimeNotSynchronized), (\"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b\", TimeNotSynchronized), (\"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71\", TimeNotSynchronized), (\"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a\", TimeNotSynchronized), (\"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\", TimeNotSynchronized), (\"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f\", TimeNotSynchronized), (\"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78\", TimeNotSynchronized), (\"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b\", TimeNotSynchronized), (\"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c\", TimeNotSynchronized), (\"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a\", TimeNotSynchronized)]","latency_us":495408,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:27.08674936Z","hostname":"buskin","pid":1661,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"1.717661972s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1d38f008-f81c-4514-bee1-6a463616f44b\", \"content-length\": \"124\", \"date\": \"Thu, 07 Mar 2024 22:45:26 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"1d38f008-f81c-4514-bee1-6a463616f44b\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 6507a4bd-9097-45d0-9787-1240feff9525, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 65847932-72ba-40f7-b62c-5adce2529400, kind: Service { id: f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf }, name: Name(\"ntp-f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 416eb39a-1cea-4752-bc22-195679c44853, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bdd869e5-e908-42b7-afb7-35517ea5ca46, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bd70b00d-6277-40dc-9a7a-524663f1a340, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 777ca252-0e2b-4952-8163-fd713097cfd8, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 0b3b3eae-a95f-4091-984a-93fccf167e79, kind: Service { id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c }, name: Name(\"external-dns-e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: ca9c5211-aa62-4394-a496-e20bf51739bc, kind: Service { id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b }, name: Name(\"external-dns-472c8e8c-ac8f-4193-921c-49dd3ceb288b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 761d7d83-4f44-4dad-ba8f-694aeae46e71, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 1eeeae19-992a-47f8-9e7b-dc93c05f0958, kind: Service { id: 761d7d83-4f44-4dad-ba8f-694aeae46e71 }, name: Name(\"nexus-761d7d83-4f44-4dad-ba8f-694aeae46e71\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a835148-a2f0-40d4-bc66-05f1d779465b, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 89eadaf9-099e-4929-ad88-e7ee2258e13d, kind: Service { id: 0a835148-a2f0-40d4-bc66-05f1d779465b }, name: Name(\"nexus-0a835148-a2f0-40d4-bc66-05f1d779465b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 72])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2bea071a-6ee2-45ba-9a69-93a0e2e5b31b, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 8c296dc4-9b38-40c8-9549-80b2967f6cdc, kind: Service { id: 2bea071a-6ee2-45ba-9a69-93a0e2e5b31b }, name: Name(\"nexus-2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8dc5039d-cb88-49dd-8ab8-2ca47e481b18, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: e79fbd89-0714-4c5d-b550-de2def94225f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 16f9d651-64fc-4e3c-a69b-bda36e263e78, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: afeebd42-9418-4a17-adf1-4ce6f3cadd20, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 5eefa476-ecad-43d0-a297-e34591893cf5, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a18cc15b-758a-467a-8a3b-ed115f6e1135, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 36c43d98-b274-450d-98d0-db7e0ee7514c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 9227e547-c76f-4bc5-b6f2-101d008d915e, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 608a3190-2550-47a2-a23f-81797b571827, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: fca244c4-2682-42f7-9297-53c69ba4caa1, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:28.806019335Z","hostname":"buskin","pid":1661,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6507a4bd-9097-45d0-9787-1240feff9525, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:28.806699556Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6507a4bd-9097-45d0-9787-1240feff9525, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:28.806713586Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:28.8299984Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:28.830042231Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:28.850622732Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:28.850664514Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 416eb39a-1cea-4752-bc22-195679c44853, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:28.872339028Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 416eb39a-1cea-4752-bc22-195679c44853, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:28.87238236Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bdd869e5-e908-42b7-afb7-35517ea5ca46, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:28.891471246Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bdd869e5-e908-42b7-afb7-35517ea5ca46, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:28.891511587Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd70b00d-6277-40dc-9a7a-524663f1a340, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:28.910897953Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bd70b00d-6277-40dc-9a7a-524663f1a340, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:28.910937434Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:28.929104683Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:28.929158884Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 777ca252-0e2b-4952-8163-fd713097cfd8, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:28.950213691Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 777ca252-0e2b-4952-8163-fd713097cfd8, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:28.950274292Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:28.968202854Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:28.968267126Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:28.988915259Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:28.98895563Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:29.008519827Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:29.008563685Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:29.028152547Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:29.028195954Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5eefa476-ecad-43d0-a297-e34591893cf5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:29.049090922Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5eefa476-ecad-43d0-a297-e34591893cf5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:29.049133693Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a18cc15b-758a-467a-8a3b-ed115f6e1135, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:29.068395667Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a18cc15b-758a-467a-8a3b-ed115f6e1135, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:29.068453035Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 36c43d98-b274-450d-98d0-db7e0ee7514c, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:29.090942176Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 36c43d98-b274-450d-98d0-db7e0ee7514c, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:29.091048718Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9227e547-c76f-4bc5-b6f2-101d008d915e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:29.110565554Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9227e547-c76f-4bc5-b6f2-101d008d915e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:29.110603439Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:29.131235958Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:29.131279393Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 608a3190-2550-47a2-a23f-81797b571827, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:29.15048654Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 608a3190-2550-47a2-a23f-81797b571827, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:29.150534578Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:29.169007578Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:29.169050367Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fca244c4-2682-42f7-9297-53c69ba4caa1, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:29.187192264Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fca244c4-2682-42f7-9297-53c69ba4caa1, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:29.187293073Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:29.205785359Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:29.205835301Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:29.206017144Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:29.206133079Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:29.302853846Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:29.30290737Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:29.302915226Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:29.302921995Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:29.302945387Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:29.302953609Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:29.302958681Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:29.302966228Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:29.302974122Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:29.302981969Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:29.302988016Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:29.302995854Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:29.303002492Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:29.303012345Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:29.303018429Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:29.303026108Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:29.303030927Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:29.303037143Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:29.303041718Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:29.303117986Z","hostname":"buskin","pid":1661,"uri":"/omicron-zones","method":"PUT","req_id":"f9398246-b71f-4ded-b4e6-2fa0ca9d9732","remote_addr":"[fd00:1122:3344:101::1]:58669","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78\", TimeNotSynchronized), (\"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4\", TimeNotSynchronized), (\"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c\", TimeNotSynchronized), (\"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1\", TimeNotSynchronized), (\"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b\", TimeNotSynchronized), (\"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b\", TimeNotSynchronized), (\"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135\", TimeNotSynchronized), (\"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e\", TimeNotSynchronized), (\"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5\", TimeNotSynchronized), (\"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b\", TimeNotSynchronized), (\"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18\", TimeNotSynchronized), (\"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f\", TimeNotSynchronized), (\"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\", TimeNotSynchronized), (\"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827\", TimeNotSynchronized), (\"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20\", TimeNotSynchronized), (\"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a\", TimeNotSynchronized), (\"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71\", TimeNotSynchronized), (\"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\", TimeNotSynchronized), (\"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a\", TimeNotSynchronized)]","latency_us":495863,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:29.303372909Z","hostname":"buskin","pid":1661,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"1.562751488s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f9398246-b71f-4ded-b4e6-2fa0ca9d9732\", \"content-length\": \"124\", \"date\": \"Thu, 07 Mar 2024 22:45:28 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f9398246-b71f-4ded-b4e6-2fa0ca9d9732\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 6507a4bd-9097-45d0-9787-1240feff9525, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 65847932-72ba-40f7-b62c-5adce2529400, kind: Service { id: f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf }, name: Name(\"ntp-f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 416eb39a-1cea-4752-bc22-195679c44853, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bdd869e5-e908-42b7-afb7-35517ea5ca46, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bd70b00d-6277-40dc-9a7a-524663f1a340, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 777ca252-0e2b-4952-8163-fd713097cfd8, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 0b3b3eae-a95f-4091-984a-93fccf167e79, kind: Service { id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c }, name: Name(\"external-dns-e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: ca9c5211-aa62-4394-a496-e20bf51739bc, kind: Service { id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b }, name: Name(\"external-dns-472c8e8c-ac8f-4193-921c-49dd3ceb288b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 761d7d83-4f44-4dad-ba8f-694aeae46e71, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 1eeeae19-992a-47f8-9e7b-dc93c05f0958, kind: Service { id: 761d7d83-4f44-4dad-ba8f-694aeae46e71 }, name: Name(\"nexus-761d7d83-4f44-4dad-ba8f-694aeae46e71\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a835148-a2f0-40d4-bc66-05f1d779465b, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 89eadaf9-099e-4929-ad88-e7ee2258e13d, kind: Service { id: 0a835148-a2f0-40d4-bc66-05f1d779465b }, name: Name(\"nexus-0a835148-a2f0-40d4-bc66-05f1d779465b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 72])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2bea071a-6ee2-45ba-9a69-93a0e2e5b31b, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 8c296dc4-9b38-40c8-9549-80b2967f6cdc, kind: Service { id: 2bea071a-6ee2-45ba-9a69-93a0e2e5b31b }, name: Name(\"nexus-2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8dc5039d-cb88-49dd-8ab8-2ca47e481b18, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: e79fbd89-0714-4c5d-b550-de2def94225f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 16f9d651-64fc-4e3c-a69b-bda36e263e78, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: afeebd42-9418-4a17-adf1-4ce6f3cadd20, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 5eefa476-ecad-43d0-a297-e34591893cf5, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a18cc15b-758a-467a-8a3b-ed115f6e1135, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 36c43d98-b274-450d-98d0-db7e0ee7514c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 9227e547-c76f-4bc5-b6f2-101d008d915e, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 608a3190-2550-47a2-a23f-81797b571827, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: fca244c4-2682-42f7-9297-53c69ba4caa1, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:30.871693809Z","hostname":"buskin","pid":1661,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6507a4bd-9097-45d0-9787-1240feff9525, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:30.872409726Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6507a4bd-9097-45d0-9787-1240feff9525, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:30.872426199Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:30.896152882Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:30.896215847Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:30.918139558Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:30.918198659Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 416eb39a-1cea-4752-bc22-195679c44853, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:30.939090031Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 416eb39a-1cea-4752-bc22-195679c44853, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:30.939141388Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bdd869e5-e908-42b7-afb7-35517ea5ca46, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:30.958223288Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bdd869e5-e908-42b7-afb7-35517ea5ca46, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:30.958275097Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd70b00d-6277-40dc-9a7a-524663f1a340, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:30.977315105Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bd70b00d-6277-40dc-9a7a-524663f1a340, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:30.977368629Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:30.996999002Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:30.997047904Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 777ca252-0e2b-4952-8163-fd713097cfd8, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:31.01575208Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 777ca252-0e2b-4952-8163-fd713097cfd8, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:31.015820585Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:31.038980994Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:31.039029145Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:31.059992874Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:31.060051263Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:31.079965122Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:31.080025143Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:31.099492979Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:31.099548022Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5eefa476-ecad-43d0-a297-e34591893cf5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:31.119500532Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5eefa476-ecad-43d0-a297-e34591893cf5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:31.119552292Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a18cc15b-758a-467a-8a3b-ed115f6e1135, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:31.139340615Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a18cc15b-758a-467a-8a3b-ed115f6e1135, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:31.139393688Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 36c43d98-b274-450d-98d0-db7e0ee7514c, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:31.158464756Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 36c43d98-b274-450d-98d0-db7e0ee7514c, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:31.158517211Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9227e547-c76f-4bc5-b6f2-101d008d915e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:31.17755361Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9227e547-c76f-4bc5-b6f2-101d008d915e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:31.177667092Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:31.196755454Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:31.196799526Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 608a3190-2550-47a2-a23f-81797b571827, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:31.215333821Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 608a3190-2550-47a2-a23f-81797b571827, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:31.215385806Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:31.235151816Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:31.235202705Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fca244c4-2682-42f7-9297-53c69ba4caa1, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:31.253930099Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fca244c4-2682-42f7-9297-53c69ba4caa1, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:31.253980632Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:31.272690609Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:31.272759967Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:31.273000815Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:31.273134929Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:31.37662589Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:31.376707323Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:31.376716211Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:31.376723599Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:31.376728258Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:31.376735055Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:31.37674083Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:31.376746249Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:31.376752606Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:31.376757903Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:31.376764212Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:31.3767707Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:31.376774937Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:31.376780084Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:31.376786545Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:31.376791916Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:31.376798301Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:31.37680672Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:31.376812196Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:31.376895072Z","hostname":"buskin","pid":1661,"uri":"/omicron-zones","method":"PUT","req_id":"17e430d8-fba3-4657-bb07-ddf2f004740d","remote_addr":"[fd00:1122:3344:101::1]:58669","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\", TimeNotSynchronized), (\"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b\", TimeNotSynchronized), (\"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78\", TimeNotSynchronized), (\"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4\", TimeNotSynchronized), (\"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f\", TimeNotSynchronized), (\"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5\", TimeNotSynchronized), (\"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c\", TimeNotSynchronized), (\"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71\", TimeNotSynchronized), (\"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18\", TimeNotSynchronized), (\"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20\", TimeNotSynchronized), (\"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135\", TimeNotSynchronized), (\"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827\", TimeNotSynchronized), (\"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a\", TimeNotSynchronized), (\"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b\", TimeNotSynchronized), (\"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a\", TimeNotSynchronized), (\"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e\", TimeNotSynchronized), (\"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b\", TimeNotSynchronized), (\"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1\", TimeNotSynchronized), (\"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\", TimeNotSynchronized)]","latency_us":503359,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:31.377110648Z","hostname":"buskin","pid":1661,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"2.601700304s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"17e430d8-fba3-4657-bb07-ddf2f004740d\", \"content-length\": \"124\", \"date\": \"Thu, 07 Mar 2024 22:45:31 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"17e430d8-fba3-4657-bb07-ddf2f004740d\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 6507a4bd-9097-45d0-9787-1240feff9525, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 65847932-72ba-40f7-b62c-5adce2529400, kind: Service { id: f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf }, name: Name(\"ntp-f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 416eb39a-1cea-4752-bc22-195679c44853, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bdd869e5-e908-42b7-afb7-35517ea5ca46, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bd70b00d-6277-40dc-9a7a-524663f1a340, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 777ca252-0e2b-4952-8163-fd713097cfd8, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 0b3b3eae-a95f-4091-984a-93fccf167e79, kind: Service { id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c }, name: Name(\"external-dns-e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: ca9c5211-aa62-4394-a496-e20bf51739bc, kind: Service { id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b }, name: Name(\"external-dns-472c8e8c-ac8f-4193-921c-49dd3ceb288b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 761d7d83-4f44-4dad-ba8f-694aeae46e71, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 1eeeae19-992a-47f8-9e7b-dc93c05f0958, kind: Service { id: 761d7d83-4f44-4dad-ba8f-694aeae46e71 }, name: Name(\"nexus-761d7d83-4f44-4dad-ba8f-694aeae46e71\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a835148-a2f0-40d4-bc66-05f1d779465b, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 89eadaf9-099e-4929-ad88-e7ee2258e13d, kind: Service { id: 0a835148-a2f0-40d4-bc66-05f1d779465b }, name: Name(\"nexus-0a835148-a2f0-40d4-bc66-05f1d779465b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 72])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2bea071a-6ee2-45ba-9a69-93a0e2e5b31b, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 8c296dc4-9b38-40c8-9549-80b2967f6cdc, kind: Service { id: 2bea071a-6ee2-45ba-9a69-93a0e2e5b31b }, name: Name(\"nexus-2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8dc5039d-cb88-49dd-8ab8-2ca47e481b18, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: e79fbd89-0714-4c5d-b550-de2def94225f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 16f9d651-64fc-4e3c-a69b-bda36e263e78, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: afeebd42-9418-4a17-adf1-4ce6f3cadd20, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 5eefa476-ecad-43d0-a297-e34591893cf5, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a18cc15b-758a-467a-8a3b-ed115f6e1135, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 36c43d98-b274-450d-98d0-db7e0ee7514c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 9227e547-c76f-4bc5-b6f2-101d008d915e, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 608a3190-2550-47a2-a23f-81797b571827, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: fca244c4-2682-42f7-9297-53c69ba4caa1, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:33.98738188Z","hostname":"buskin","pid":1661,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6507a4bd-9097-45d0-9787-1240feff9525, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:33.988309667Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6507a4bd-9097-45d0-9787-1240feff9525, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:33.988322614Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:34.011729558Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:34.011794847Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:34.034224191Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:34.034287558Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 416eb39a-1cea-4752-bc22-195679c44853, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:34.056431178Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 416eb39a-1cea-4752-bc22-195679c44853, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:34.056488264Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bdd869e5-e908-42b7-afb7-35517ea5ca46, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:34.077602232Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bdd869e5-e908-42b7-afb7-35517ea5ca46, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:34.077655494Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd70b00d-6277-40dc-9a7a-524663f1a340, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:34.096903763Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bd70b00d-6277-40dc-9a7a-524663f1a340, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:34.096959396Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:34.117200852Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:34.119446427Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 777ca252-0e2b-4952-8163-fd713097cfd8, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:34.136783188Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 777ca252-0e2b-4952-8163-fd713097cfd8, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:34.13683718Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:34.155260736Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:34.155312966Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:34.175024041Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:34.175076533Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:34.192938644Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:34.192995599Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:34.212358685Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:34.212417609Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5eefa476-ecad-43d0-a297-e34591893cf5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:34.231163427Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5eefa476-ecad-43d0-a297-e34591893cf5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:34.231218094Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a18cc15b-758a-467a-8a3b-ed115f6e1135, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:34.249415977Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a18cc15b-758a-467a-8a3b-ed115f6e1135, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:34.249476054Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 36c43d98-b274-450d-98d0-db7e0ee7514c, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:34.269531997Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 36c43d98-b274-450d-98d0-db7e0ee7514c, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:34.269590321Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9227e547-c76f-4bc5-b6f2-101d008d915e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:34.288437291Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9227e547-c76f-4bc5-b6f2-101d008d915e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:34.288496627Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:34.30675938Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:34.30680919Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 608a3190-2550-47a2-a23f-81797b571827, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:34.324444275Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 608a3190-2550-47a2-a23f-81797b571827, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:34.324499186Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:34.341872472Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:34.341931873Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fca244c4-2682-42f7-9297-53c69ba4caa1, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:34.360336095Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fca244c4-2682-42f7-9297-53c69ba4caa1, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:34.360385643Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:34.37996188Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:34.380022941Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:34.380220788Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:34.380430081Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:34.479629801Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:34.479666215Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:34.479671803Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:34.479676434Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:34.479683962Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:34.47968834Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:34.479691913Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:34.479696038Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:34.479700313Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:34.479704644Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:34.479708882Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:34.479713044Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:34.479717151Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:34.479721314Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:34.479734289Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:34.479739014Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:34.479743767Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:34.479747855Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:34.479752384Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:34.47986934Z","hostname":"buskin","pid":1661,"uri":"/omicron-zones","method":"PUT","req_id":"e97819b0-54bc-44ae-8b98-0881e63c4860","remote_addr":"[fd00:1122:3344:101::1]:58669","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78\", TimeNotSynchronized), (\"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a\", TimeNotSynchronized), (\"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18\", TimeNotSynchronized), (\"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f\", TimeNotSynchronized), (\"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4\", TimeNotSynchronized), (\"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827\", TimeNotSynchronized), (\"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1\", TimeNotSynchronized), (\"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5\", TimeNotSynchronized), (\"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e\", TimeNotSynchronized), (\"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\", TimeNotSynchronized), (\"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20\", TimeNotSynchronized), (\"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b\", TimeNotSynchronized), (\"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71\", TimeNotSynchronized), (\"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b\", TimeNotSynchronized), (\"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135\", TimeNotSynchronized), (\"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\", TimeNotSynchronized), (\"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c\", TimeNotSynchronized), (\"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b\", TimeNotSynchronized), (\"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a\", TimeNotSynchronized)]","latency_us":490470,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:34.480088093Z","hostname":"buskin","pid":1661,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"1.409050473s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e97819b0-54bc-44ae-8b98-0881e63c4860\", \"content-length\": \"124\", \"date\": \"Thu, 07 Mar 2024 22:45:33 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e97819b0-54bc-44ae-8b98-0881e63c4860\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 6507a4bd-9097-45d0-9787-1240feff9525, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 65847932-72ba-40f7-b62c-5adce2529400, kind: Service { id: f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf }, name: Name(\"ntp-f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 416eb39a-1cea-4752-bc22-195679c44853, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bdd869e5-e908-42b7-afb7-35517ea5ca46, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bd70b00d-6277-40dc-9a7a-524663f1a340, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 777ca252-0e2b-4952-8163-fd713097cfd8, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 0b3b3eae-a95f-4091-984a-93fccf167e79, kind: Service { id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c }, name: Name(\"external-dns-e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: ca9c5211-aa62-4394-a496-e20bf51739bc, kind: Service { id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b }, name: Name(\"external-dns-472c8e8c-ac8f-4193-921c-49dd3ceb288b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 761d7d83-4f44-4dad-ba8f-694aeae46e71, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 1eeeae19-992a-47f8-9e7b-dc93c05f0958, kind: Service { id: 761d7d83-4f44-4dad-ba8f-694aeae46e71 }, name: Name(\"nexus-761d7d83-4f44-4dad-ba8f-694aeae46e71\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a835148-a2f0-40d4-bc66-05f1d779465b, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 89eadaf9-099e-4929-ad88-e7ee2258e13d, kind: Service { id: 0a835148-a2f0-40d4-bc66-05f1d779465b }, name: Name(\"nexus-0a835148-a2f0-40d4-bc66-05f1d779465b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 72])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2bea071a-6ee2-45ba-9a69-93a0e2e5b31b, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 8c296dc4-9b38-40c8-9549-80b2967f6cdc, kind: Service { id: 2bea071a-6ee2-45ba-9a69-93a0e2e5b31b }, name: Name(\"nexus-2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8dc5039d-cb88-49dd-8ab8-2ca47e481b18, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: e79fbd89-0714-4c5d-b550-de2def94225f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 16f9d651-64fc-4e3c-a69b-bda36e263e78, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: afeebd42-9418-4a17-adf1-4ce6f3cadd20, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 5eefa476-ecad-43d0-a297-e34591893cf5, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a18cc15b-758a-467a-8a3b-ed115f6e1135, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 36c43d98-b274-450d-98d0-db7e0ee7514c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 9227e547-c76f-4bc5-b6f2-101d008d915e, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 608a3190-2550-47a2-a23f-81797b571827, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: fca244c4-2682-42f7-9297-53c69ba4caa1, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:35.894324627Z","hostname":"buskin","pid":1661,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6507a4bd-9097-45d0-9787-1240feff9525, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:35.895080751Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6507a4bd-9097-45d0-9787-1240feff9525, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:35.895093717Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:35.918935668Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:35.919001895Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:35.941329731Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:35.941392883Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 416eb39a-1cea-4752-bc22-195679c44853, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:35.959643589Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 416eb39a-1cea-4752-bc22-195679c44853, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:35.95969986Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bdd869e5-e908-42b7-afb7-35517ea5ca46, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:35.978550478Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bdd869e5-e908-42b7-afb7-35517ea5ca46, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:35.978671928Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd70b00d-6277-40dc-9a7a-524663f1a340, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:35.996384978Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bd70b00d-6277-40dc-9a7a-524663f1a340, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:35.996453398Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:36.015534265Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:36.015584732Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 777ca252-0e2b-4952-8163-fd713097cfd8, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:36.034653144Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 777ca252-0e2b-4952-8163-fd713097cfd8, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:36.034702909Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:36.053242243Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:36.053290151Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:36.074512968Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:36.074563961Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:36.0934113Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:36.093462021Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:36.11159987Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:36.111644975Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5eefa476-ecad-43d0-a297-e34591893cf5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:36.129924748Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5eefa476-ecad-43d0-a297-e34591893cf5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:36.130040711Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a18cc15b-758a-467a-8a3b-ed115f6e1135, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:36.149180793Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a18cc15b-758a-467a-8a3b-ed115f6e1135, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:36.149232789Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 36c43d98-b274-450d-98d0-db7e0ee7514c, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:36.16971236Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 36c43d98-b274-450d-98d0-db7e0ee7514c, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:36.169767317Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9227e547-c76f-4bc5-b6f2-101d008d915e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:36.189274873Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9227e547-c76f-4bc5-b6f2-101d008d915e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:36.189325715Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:36.208683152Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:36.20874741Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 608a3190-2550-47a2-a23f-81797b571827, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:36.228318435Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 608a3190-2550-47a2-a23f-81797b571827, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:36.228376065Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:36.248459476Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:36.248510918Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fca244c4-2682-42f7-9297-53c69ba4caa1, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:36.26761975Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fca244c4-2682-42f7-9297-53c69ba4caa1, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:36.267670217Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:36.286232749Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:36.286302089Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:36.286584648Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:36.286716305Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:36.387589034Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:36.387655636Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:36.387737692Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:36.387747724Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:36.387756367Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:36.387763362Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:36.387770018Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:36.38777495Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:36.387779778Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:36.387784653Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:36.387792546Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:36.387809104Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:36.387816041Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:36.387822754Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:36.387827414Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:36.387833873Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:36.387838373Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:36.387843051Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:36.387847973Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:36.387864118Z","hostname":"buskin","pid":1661,"uri":"/omicron-zones","method":"PUT","req_id":"50679b11-32b0-49b7-b7bf-faf51b9f15b3","remote_addr":"[fd00:1122:3344:101::1]:58669","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20\", TimeNotSynchronized), (\"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c\", TimeNotSynchronized), (\"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1\", TimeNotSynchronized), (\"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e\", TimeNotSynchronized), (\"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a\", TimeNotSynchronized), (\"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5\", TimeNotSynchronized), (\"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78\", TimeNotSynchronized), (\"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b\", TimeNotSynchronized), (\"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827\", TimeNotSynchronized), (\"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a\", TimeNotSynchronized), (\"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4\", TimeNotSynchronized), (\"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b\", TimeNotSynchronized), (\"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f\", TimeNotSynchronized), (\"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71\", TimeNotSynchronized), (\"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\", TimeNotSynchronized), (\"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\", TimeNotSynchronized), (\"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18\", TimeNotSynchronized), (\"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135\", TimeNotSynchronized), (\"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b\", TimeNotSynchronized)]","latency_us":491658,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:36.388480346Z","hostname":"buskin","pid":1661,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"3.012927985s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"50679b11-32b0-49b7-b7bf-faf51b9f15b3\", \"content-length\": \"124\", \"date\": \"Thu, 07 Mar 2024 22:45:35 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"50679b11-32b0-49b7-b7bf-faf51b9f15b3\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 6507a4bd-9097-45d0-9787-1240feff9525, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 65847932-72ba-40f7-b62c-5adce2529400, kind: Service { id: f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf }, name: Name(\"ntp-f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 416eb39a-1cea-4752-bc22-195679c44853, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bdd869e5-e908-42b7-afb7-35517ea5ca46, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bd70b00d-6277-40dc-9a7a-524663f1a340, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 777ca252-0e2b-4952-8163-fd713097cfd8, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 0b3b3eae-a95f-4091-984a-93fccf167e79, kind: Service { id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c }, name: Name(\"external-dns-e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: ca9c5211-aa62-4394-a496-e20bf51739bc, kind: Service { id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b }, name: Name(\"external-dns-472c8e8c-ac8f-4193-921c-49dd3ceb288b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 761d7d83-4f44-4dad-ba8f-694aeae46e71, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 1eeeae19-992a-47f8-9e7b-dc93c05f0958, kind: Service { id: 761d7d83-4f44-4dad-ba8f-694aeae46e71 }, name: Name(\"nexus-761d7d83-4f44-4dad-ba8f-694aeae46e71\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a835148-a2f0-40d4-bc66-05f1d779465b, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 89eadaf9-099e-4929-ad88-e7ee2258e13d, kind: Service { id: 0a835148-a2f0-40d4-bc66-05f1d779465b }, name: Name(\"nexus-0a835148-a2f0-40d4-bc66-05f1d779465b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 72])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2bea071a-6ee2-45ba-9a69-93a0e2e5b31b, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 8c296dc4-9b38-40c8-9549-80b2967f6cdc, kind: Service { id: 2bea071a-6ee2-45ba-9a69-93a0e2e5b31b }, name: Name(\"nexus-2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8dc5039d-cb88-49dd-8ab8-2ca47e481b18, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: e79fbd89-0714-4c5d-b550-de2def94225f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 16f9d651-64fc-4e3c-a69b-bda36e263e78, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: afeebd42-9418-4a17-adf1-4ce6f3cadd20, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 5eefa476-ecad-43d0-a297-e34591893cf5, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a18cc15b-758a-467a-8a3b-ed115f6e1135, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 36c43d98-b274-450d-98d0-db7e0ee7514c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 9227e547-c76f-4bc5-b6f2-101d008d915e, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 608a3190-2550-47a2-a23f-81797b571827, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: fca244c4-2682-42f7-9297-53c69ba4caa1, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:39.410801701Z","hostname":"buskin","pid":1661,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6507a4bd-9097-45d0-9787-1240feff9525, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:39.41160808Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6507a4bd-9097-45d0-9787-1240feff9525, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:39.411626127Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:39.435615381Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:39.435674942Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:39.455871369Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:39.455923581Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 416eb39a-1cea-4752-bc22-195679c44853, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:39.475778995Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 416eb39a-1cea-4752-bc22-195679c44853, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:39.475828872Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bdd869e5-e908-42b7-afb7-35517ea5ca46, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:39.496637335Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bdd869e5-e908-42b7-afb7-35517ea5ca46, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:39.49670344Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd70b00d-6277-40dc-9a7a-524663f1a340, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:39.517178603Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bd70b00d-6277-40dc-9a7a-524663f1a340, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:39.517230804Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:39.535538782Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:39.535585687Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 777ca252-0e2b-4952-8163-fd713097cfd8, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:39.556599192Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 777ca252-0e2b-4952-8163-fd713097cfd8, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:39.556649443Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:39.575830313Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:39.575885598Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:39.595507663Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:39.595562048Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:39.615311372Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:39.615361679Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:39.634538561Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:39.634591532Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5eefa476-ecad-43d0-a297-e34591893cf5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:39.65344231Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5eefa476-ecad-43d0-a297-e34591893cf5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:39.653571795Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a18cc15b-758a-467a-8a3b-ed115f6e1135, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:39.672381809Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a18cc15b-758a-467a-8a3b-ed115f6e1135, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:39.672436588Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 36c43d98-b274-450d-98d0-db7e0ee7514c, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:39.691262681Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 36c43d98-b274-450d-98d0-db7e0ee7514c, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:39.691312361Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9227e547-c76f-4bc5-b6f2-101d008d915e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:39.709924955Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9227e547-c76f-4bc5-b6f2-101d008d915e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:39.709981806Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:39.730276115Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:39.730331195Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 608a3190-2550-47a2-a23f-81797b571827, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:39.750654206Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 608a3190-2550-47a2-a23f-81797b571827, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:39.750703426Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:39.769575774Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:39.769623756Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fca244c4-2682-42f7-9297-53c69ba4caa1, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:39.78884122Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fca244c4-2682-42f7-9297-53c69ba4caa1, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:39.788965603Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:39.80805156Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:39.80811684Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:39.808318858Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:39.808464954Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:39.905055816Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:39.905118237Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:39.905128221Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:39.905135177Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:39.905143334Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:39.90515795Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:39.905231981Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:39.90523694Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:39.905251772Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:39.9052566Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:39.905261513Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:39.905273363Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:39.905281857Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:39.905288204Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:39.905293014Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:39.905303645Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:39.905308314Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:39.905313077Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:39.90531782Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:39.905324205Z","hostname":"buskin","pid":1661,"uri":"/omicron-zones","method":"PUT","req_id":"6713a6ad-0aee-42d3-ac90-522c214d48ad","remote_addr":"[fd00:1122:3344:101::1]:58669","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5\", TimeNotSynchronized), (\"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71\", TimeNotSynchronized), (\"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a\", TimeNotSynchronized), (\"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18\", TimeNotSynchronized), (\"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4\", TimeNotSynchronized), (\"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\", TimeNotSynchronized), (\"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135\", TimeNotSynchronized), (\"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c\", TimeNotSynchronized), (\"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b\", TimeNotSynchronized), (\"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b\", TimeNotSynchronized), (\"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e\", TimeNotSynchronized), (\"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b\", TimeNotSynchronized), (\"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\", TimeNotSynchronized), (\"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f\", TimeNotSynchronized), (\"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20\", TimeNotSynchronized), (\"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827\", TimeNotSynchronized), (\"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a\", TimeNotSynchronized), (\"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1\", TimeNotSynchronized), (\"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78\", TimeNotSynchronized)]","latency_us":492576,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:39.905593004Z","hostname":"buskin","pid":1661,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"3.847076155s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6713a6ad-0aee-42d3-ac90-522c214d48ad\", \"content-length\": \"124\", \"date\": \"Thu, 07 Mar 2024 22:45:39 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6713a6ad-0aee-42d3-ac90-522c214d48ad\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:43.119145389Z","hostname":"buskin","pid":1661,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:57468"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:43.119273193Z","hostname":"buskin","pid":1661,"uri":"/baseboard","method":"GET","req_id":"31b2e7f5-80e1-405c-89f4-f24a291f2e69","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:57468","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":64,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 6507a4bd-9097-45d0-9787-1240feff9525, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 65847932-72ba-40f7-b62c-5adce2529400, kind: Service { id: f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf }, name: Name(\"ntp-f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 416eb39a-1cea-4752-bc22-195679c44853, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bdd869e5-e908-42b7-afb7-35517ea5ca46, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bd70b00d-6277-40dc-9a7a-524663f1a340, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 777ca252-0e2b-4952-8163-fd713097cfd8, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 0b3b3eae-a95f-4091-984a-93fccf167e79, kind: Service { id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c }, name: Name(\"external-dns-e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: ca9c5211-aa62-4394-a496-e20bf51739bc, kind: Service { id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b }, name: Name(\"external-dns-472c8e8c-ac8f-4193-921c-49dd3ceb288b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 761d7d83-4f44-4dad-ba8f-694aeae46e71, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 1eeeae19-992a-47f8-9e7b-dc93c05f0958, kind: Service { id: 761d7d83-4f44-4dad-ba8f-694aeae46e71 }, name: Name(\"nexus-761d7d83-4f44-4dad-ba8f-694aeae46e71\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a835148-a2f0-40d4-bc66-05f1d779465b, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 89eadaf9-099e-4929-ad88-e7ee2258e13d, kind: Service { id: 0a835148-a2f0-40d4-bc66-05f1d779465b }, name: Name(\"nexus-0a835148-a2f0-40d4-bc66-05f1d779465b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 72])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2bea071a-6ee2-45ba-9a69-93a0e2e5b31b, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 8c296dc4-9b38-40c8-9549-80b2967f6cdc, kind: Service { id: 2bea071a-6ee2-45ba-9a69-93a0e2e5b31b }, name: Name(\"nexus-2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8dc5039d-cb88-49dd-8ab8-2ca47e481b18, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: e79fbd89-0714-4c5d-b550-de2def94225f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 16f9d651-64fc-4e3c-a69b-bda36e263e78, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: afeebd42-9418-4a17-adf1-4ce6f3cadd20, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 5eefa476-ecad-43d0-a297-e34591893cf5, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a18cc15b-758a-467a-8a3b-ed115f6e1135, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 36c43d98-b274-450d-98d0-db7e0ee7514c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 9227e547-c76f-4bc5-b6f2-101d008d915e, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 608a3190-2550-47a2-a23f-81797b571827, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: fca244c4-2682-42f7-9297-53c69ba4caa1, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:43.76220312Z","hostname":"buskin","pid":1661,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6507a4bd-9097-45d0-9787-1240feff9525, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:43.762913582Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6507a4bd-9097-45d0-9787-1240feff9525, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:43.762930032Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:43.785177074Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:43.785238696Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:43.805135167Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:43.805196919Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 416eb39a-1cea-4752-bc22-195679c44853, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:43.82542952Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 416eb39a-1cea-4752-bc22-195679c44853, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:43.825483871Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bdd869e5-e908-42b7-afb7-35517ea5ca46, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:43.844577638Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bdd869e5-e908-42b7-afb7-35517ea5ca46, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:43.84463045Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd70b00d-6277-40dc-9a7a-524663f1a340, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:43.863721196Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bd70b00d-6277-40dc-9a7a-524663f1a340, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:43.863779098Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:43.881841983Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:43.881905315Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 777ca252-0e2b-4952-8163-fd713097cfd8, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:43.901028843Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 777ca252-0e2b-4952-8163-fd713097cfd8, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:43.901084575Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:43.919188311Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:43.919245833Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:43.939038921Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:43.939100143Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:43.959786968Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:43.95985938Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:43.979316177Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:43.979373769Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5eefa476-ecad-43d0-a297-e34591893cf5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:43.997245269Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5eefa476-ecad-43d0-a297-e34591893cf5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:43.99729785Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a18cc15b-758a-467a-8a3b-ed115f6e1135, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:44.016759968Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a18cc15b-758a-467a-8a3b-ed115f6e1135, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:44.01681564Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 36c43d98-b274-450d-98d0-db7e0ee7514c, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:44.036435123Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 36c43d98-b274-450d-98d0-db7e0ee7514c, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:44.036496754Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9227e547-c76f-4bc5-b6f2-101d008d915e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:44.056091516Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9227e547-c76f-4bc5-b6f2-101d008d915e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:44.056145498Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:44.075607406Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:44.075666517Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 608a3190-2550-47a2-a23f-81797b571827, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:44.095240059Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 608a3190-2550-47a2-a23f-81797b571827, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:44.095301821Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:44.114640074Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:44.114697186Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fca244c4-2682-42f7-9297-53c69ba4caa1, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:44.135176805Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fca244c4-2682-42f7-9297-53c69ba4caa1, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:44.135231127Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:44.154251541Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:44.154318713Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:44.154525449Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:44.154672134Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:44.252795907Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:44.252864289Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:44.25288564Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:44.25289395Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:44.25290119Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:44.25290807Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:44.252915331Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:44.252920881Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:44.252929791Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:44.252954602Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:44.252961972Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:44.252970963Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:44.252978033Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:44.252986773Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:44.252995313Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:44.253004184Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:44.253009864Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:44.253027354Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:44.253033074Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:44.253122817Z","hostname":"buskin","pid":1661,"uri":"/omicron-zones","method":"PUT","req_id":"7bef2a4f-7655-4916-99c1-841513189712","remote_addr":"[fd00:1122:3344:101::1]:58669","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f\", TimeNotSynchronized), (\"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a\", TimeNotSynchronized), (\"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78\", TimeNotSynchronized), (\"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20\", TimeNotSynchronized), (\"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b\", TimeNotSynchronized), (\"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71\", TimeNotSynchronized), (\"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135\", TimeNotSynchronized), (\"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1\", TimeNotSynchronized), (\"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18\", TimeNotSynchronized), (\"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c\", TimeNotSynchronized), (\"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827\", TimeNotSynchronized), (\"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\", TimeNotSynchronized), (\"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b\", TimeNotSynchronized), (\"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\", TimeNotSynchronized), (\"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4\", TimeNotSynchronized), (\"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5\", TimeNotSynchronized), (\"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b\", TimeNotSynchronized), (\"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a\", TimeNotSynchronized), (\"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e\", TimeNotSynchronized)]","latency_us":490420,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:44.253359484Z","hostname":"buskin","pid":1661,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"5.501339652s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7bef2a4f-7655-4916-99c1-841513189712\", \"content-length\": \"124\", \"date\": \"Thu, 07 Mar 2024 22:45:44 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"7bef2a4f-7655-4916-99c1-841513189712\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 6507a4bd-9097-45d0-9787-1240feff9525, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 65847932-72ba-40f7-b62c-5adce2529400, kind: Service { id: f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf }, name: Name(\"ntp-f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 416eb39a-1cea-4752-bc22-195679c44853, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bdd869e5-e908-42b7-afb7-35517ea5ca46, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bd70b00d-6277-40dc-9a7a-524663f1a340, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 777ca252-0e2b-4952-8163-fd713097cfd8, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 0b3b3eae-a95f-4091-984a-93fccf167e79, kind: Service { id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c }, name: Name(\"external-dns-e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: ca9c5211-aa62-4394-a496-e20bf51739bc, kind: Service { id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b }, name: Name(\"external-dns-472c8e8c-ac8f-4193-921c-49dd3ceb288b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 761d7d83-4f44-4dad-ba8f-694aeae46e71, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 1eeeae19-992a-47f8-9e7b-dc93c05f0958, kind: Service { id: 761d7d83-4f44-4dad-ba8f-694aeae46e71 }, name: Name(\"nexus-761d7d83-4f44-4dad-ba8f-694aeae46e71\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a835148-a2f0-40d4-bc66-05f1d779465b, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 89eadaf9-099e-4929-ad88-e7ee2258e13d, kind: Service { id: 0a835148-a2f0-40d4-bc66-05f1d779465b }, name: Name(\"nexus-0a835148-a2f0-40d4-bc66-05f1d779465b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 72])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2bea071a-6ee2-45ba-9a69-93a0e2e5b31b, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 8c296dc4-9b38-40c8-9549-80b2967f6cdc, kind: Service { id: 2bea071a-6ee2-45ba-9a69-93a0e2e5b31b }, name: Name(\"nexus-2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8dc5039d-cb88-49dd-8ab8-2ca47e481b18, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: e79fbd89-0714-4c5d-b550-de2def94225f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 16f9d651-64fc-4e3c-a69b-bda36e263e78, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: afeebd42-9418-4a17-adf1-4ce6f3cadd20, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 5eefa476-ecad-43d0-a297-e34591893cf5, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a18cc15b-758a-467a-8a3b-ed115f6e1135, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 36c43d98-b274-450d-98d0-db7e0ee7514c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 9227e547-c76f-4bc5-b6f2-101d008d915e, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 608a3190-2550-47a2-a23f-81797b571827, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: fca244c4-2682-42f7-9297-53c69ba4caa1, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:49.756032636Z","hostname":"buskin","pid":1661,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6507a4bd-9097-45d0-9787-1240feff9525, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:49.756706346Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6507a4bd-9097-45d0-9787-1240feff9525, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:49.756723747Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:49.780109563Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:49.780182965Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:49.800153728Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:49.80022597Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 416eb39a-1cea-4752-bc22-195679c44853, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:49.820547274Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 416eb39a-1cea-4752-bc22-195679c44853, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:49.820602386Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bdd869e5-e908-42b7-afb7-35517ea5ca46, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:49.841470156Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bdd869e5-e908-42b7-afb7-35517ea5ca46, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:49.844047714Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd70b00d-6277-40dc-9a7a-524663f1a340, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:49.860565723Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bd70b00d-6277-40dc-9a7a-524663f1a340, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:49.860619975Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:49.88032011Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:49.880378271Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 777ca252-0e2b-4952-8163-fd713097cfd8, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:49.89853007Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 777ca252-0e2b-4952-8163-fd713097cfd8, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:49.901077907Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:49.917692998Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:49.91775235Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:49.936193957Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:49.936253319Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:49.95450091Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:49.957023696Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:49.974568016Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:49.977061291Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5eefa476-ecad-43d0-a297-e34591893cf5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:49.993152257Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5eefa476-ecad-43d0-a297-e34591893cf5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:49.993209109Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a18cc15b-758a-467a-8a3b-ed115f6e1135, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:50.012594395Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a18cc15b-758a-467a-8a3b-ed115f6e1135, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:50.012668197Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 36c43d98-b274-450d-98d0-db7e0ee7514c, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:50.034365322Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 36c43d98-b274-450d-98d0-db7e0ee7514c, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:50.034414054Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9227e547-c76f-4bc5-b6f2-101d008d915e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:50.053031446Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9227e547-c76f-4bc5-b6f2-101d008d915e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:50.053086558Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:50.071505784Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:50.071561006Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 608a3190-2550-47a2-a23f-81797b571827, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:50.089403775Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 608a3190-2550-47a2-a23f-81797b571827, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:50.089469617Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:50.108734978Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:50.111280245Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fca244c4-2682-42f7-9297-53c69ba4caa1, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:50.1279933Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fca244c4-2682-42f7-9297-53c69ba4caa1, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:50.128038271Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:50.147598502Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:50.147663454Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:50.147875881Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:50.148017915Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:50.242263501Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:50.242326393Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:50.242335563Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:50.242342274Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:50.242350904Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:50.242360994Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:50.242370664Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:50.242380295Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:50.242390315Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:50.242400295Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:50.242414326Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:50.242424256Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:50.242434257Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:50.242442087Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:50.242464907Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:50.242472747Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:50.242488678Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:50.242498798Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:50.242505239Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:50.24254923Z","hostname":"buskin","pid":1661,"uri":"/omicron-zones","method":"PUT","req_id":"2bf686f3-abc2-468d-adf1-1096ce3df9f7","remote_addr":"[fd00:1122:3344:101::1]:58669","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b\", TimeNotSynchronized), (\"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135\", TimeNotSynchronized), (\"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\", TimeNotSynchronized), (\"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a\", TimeNotSynchronized), (\"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78\", TimeNotSynchronized), (\"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71\", TimeNotSynchronized), (\"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4\", TimeNotSynchronized), (\"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5\", TimeNotSynchronized), (\"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\", TimeNotSynchronized), (\"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18\", TimeNotSynchronized), (\"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f\", TimeNotSynchronized), (\"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b\", TimeNotSynchronized), (\"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e\", TimeNotSynchronized), (\"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b\", TimeNotSynchronized), (\"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a\", TimeNotSynchronized), (\"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1\", TimeNotSynchronized), (\"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827\", TimeNotSynchronized), (\"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c\", TimeNotSynchronized), (\"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20\", TimeNotSynchronized)]","latency_us":486076,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:50.242786627Z","hostname":"buskin","pid":1661,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"5.750930222s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2bf686f3-abc2-468d-adf1-1096ce3df9f7\", \"content-length\": \"124\", \"date\": \"Thu, 07 Mar 2024 22:45:49 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"2bf686f3-abc2-468d-adf1-1096ce3df9f7\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 6507a4bd-9097-45d0-9787-1240feff9525, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 65847932-72ba-40f7-b62c-5adce2529400, kind: Service { id: f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf }, name: Name(\"ntp-f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 416eb39a-1cea-4752-bc22-195679c44853, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bdd869e5-e908-42b7-afb7-35517ea5ca46, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bd70b00d-6277-40dc-9a7a-524663f1a340, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 777ca252-0e2b-4952-8163-fd713097cfd8, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 0b3b3eae-a95f-4091-984a-93fccf167e79, kind: Service { id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c }, name: Name(\"external-dns-e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: ca9c5211-aa62-4394-a496-e20bf51739bc, kind: Service { id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b }, name: Name(\"external-dns-472c8e8c-ac8f-4193-921c-49dd3ceb288b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 761d7d83-4f44-4dad-ba8f-694aeae46e71, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 1eeeae19-992a-47f8-9e7b-dc93c05f0958, kind: Service { id: 761d7d83-4f44-4dad-ba8f-694aeae46e71 }, name: Name(\"nexus-761d7d83-4f44-4dad-ba8f-694aeae46e71\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a835148-a2f0-40d4-bc66-05f1d779465b, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 89eadaf9-099e-4929-ad88-e7ee2258e13d, kind: Service { id: 0a835148-a2f0-40d4-bc66-05f1d779465b }, name: Name(\"nexus-0a835148-a2f0-40d4-bc66-05f1d779465b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 72])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2bea071a-6ee2-45ba-9a69-93a0e2e5b31b, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 8c296dc4-9b38-40c8-9549-80b2967f6cdc, kind: Service { id: 2bea071a-6ee2-45ba-9a69-93a0e2e5b31b }, name: Name(\"nexus-2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8dc5039d-cb88-49dd-8ab8-2ca47e481b18, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: e79fbd89-0714-4c5d-b550-de2def94225f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 16f9d651-64fc-4e3c-a69b-bda36e263e78, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: afeebd42-9418-4a17-adf1-4ce6f3cadd20, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 5eefa476-ecad-43d0-a297-e34591893cf5, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a18cc15b-758a-467a-8a3b-ed115f6e1135, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 36c43d98-b274-450d-98d0-db7e0ee7514c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 9227e547-c76f-4bc5-b6f2-101d008d915e, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 608a3190-2550-47a2-a23f-81797b571827, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: fca244c4-2682-42f7-9297-53c69ba4caa1, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:55.994641014Z","hostname":"buskin","pid":1661,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6507a4bd-9097-45d0-9787-1240feff9525, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:55.995383497Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6507a4bd-9097-45d0-9787-1240feff9525, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:55.995405707Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:56.019780113Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:56.019843735Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:56.04084076Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:56.040896071Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 416eb39a-1cea-4752-bc22-195679c44853, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:56.06007201Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 416eb39a-1cea-4752-bc22-195679c44853, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:56.060124712Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bdd869e5-e908-42b7-afb7-35517ea5ca46, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:56.078254039Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bdd869e5-e908-42b7-afb7-35517ea5ca46, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:56.078312541Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd70b00d-6277-40dc-9a7a-524663f1a340, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:56.096375377Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bd70b00d-6277-40dc-9a7a-524663f1a340, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:56.096435349Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:56.113959368Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:56.114009019Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 777ca252-0e2b-4952-8163-fd713097cfd8, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:56.1315785Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 777ca252-0e2b-4952-8163-fd713097cfd8, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:56.131640142Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:56.149285635Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:56.149334926Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:56.167428133Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:56.167479965Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:56.185662374Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:56.185720885Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:56.204368559Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:56.20442774Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5eefa476-ecad-43d0-a297-e34591893cf5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:56.222721453Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5eefa476-ecad-43d0-a297-e34591893cf5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:56.222773744Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a18cc15b-758a-467a-8a3b-ed115f6e1135, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:56.241355966Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a18cc15b-758a-467a-8a3b-ed115f6e1135, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:56.241408717Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 36c43d98-b274-450d-98d0-db7e0ee7514c, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:56.259584456Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 36c43d98-b274-450d-98d0-db7e0ee7514c, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:56.259640398Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9227e547-c76f-4bc5-b6f2-101d008d915e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:56.277994162Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9227e547-c76f-4bc5-b6f2-101d008d915e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:56.278054274Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:56.296354117Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:56.298858892Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 608a3190-2550-47a2-a23f-81797b571827, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:56.31400391Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 608a3190-2550-47a2-a23f-81797b571827, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:56.314069502Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:56.332626932Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:56.332683504Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fca244c4-2682-42f7-9297-53c69ba4caa1, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:56.350159512Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fca244c4-2682-42f7-9297-53c69ba4caa1, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:56.350218594Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:56.368336721Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:56.368413453Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:56.36864855Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:56.368798105Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:56.465673731Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:56.465724452Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:56.465730612Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:56.465736242Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:56.465741683Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:56.465753173Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:56.465758863Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:56.465765463Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:56.465770963Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:56.465777284Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:56.465783684Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:56.465803355Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:56.465807865Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:56.465812115Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:56.465816245Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:56.465821495Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:56.465826795Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:56.465837176Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:56.465843876Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:45:56.465893287Z","hostname":"buskin","pid":1661,"uri":"/omicron-zones","method":"PUT","req_id":"ab6866a5-9364-4e8c-ae36-e04bbf3eef2a","remote_addr":"[fd00:1122:3344:101::1]:58669","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e\", TimeNotSynchronized), (\"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135\", TimeNotSynchronized), (\"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827\", TimeNotSynchronized), (\"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78\", TimeNotSynchronized), (\"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b\", TimeNotSynchronized), (\"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b\", TimeNotSynchronized), (\"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4\", TimeNotSynchronized), (\"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20\", TimeNotSynchronized), (\"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5\", TimeNotSynchronized), (\"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18\", TimeNotSynchronized), (\"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a\", TimeNotSynchronized), (\"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1\", TimeNotSynchronized), (\"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\", TimeNotSynchronized), (\"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b\", TimeNotSynchronized), (\"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c\", TimeNotSynchronized), (\"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71\", TimeNotSynchronized), (\"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f\", TimeNotSynchronized), (\"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a\", TimeNotSynchronized), (\"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\", TimeNotSynchronized)]","latency_us":470789,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:45:56.466130734Z","hostname":"buskin","pid":1661,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"3.619659164s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ab6866a5-9364-4e8c-ae36-e04bbf3eef2a\", \"content-length\": \"124\", \"date\": \"Thu, 07 Mar 2024 22:45:56 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ab6866a5-9364-4e8c-ae36-e04bbf3eef2a\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 6507a4bd-9097-45d0-9787-1240feff9525, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 65847932-72ba-40f7-b62c-5adce2529400, kind: Service { id: f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf }, name: Name(\"ntp-f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 416eb39a-1cea-4752-bc22-195679c44853, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bdd869e5-e908-42b7-afb7-35517ea5ca46, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bd70b00d-6277-40dc-9a7a-524663f1a340, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 777ca252-0e2b-4952-8163-fd713097cfd8, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 0b3b3eae-a95f-4091-984a-93fccf167e79, kind: Service { id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c }, name: Name(\"external-dns-e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: ca9c5211-aa62-4394-a496-e20bf51739bc, kind: Service { id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b }, name: Name(\"external-dns-472c8e8c-ac8f-4193-921c-49dd3ceb288b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 761d7d83-4f44-4dad-ba8f-694aeae46e71, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 1eeeae19-992a-47f8-9e7b-dc93c05f0958, kind: Service { id: 761d7d83-4f44-4dad-ba8f-694aeae46e71 }, name: Name(\"nexus-761d7d83-4f44-4dad-ba8f-694aeae46e71\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a835148-a2f0-40d4-bc66-05f1d779465b, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 89eadaf9-099e-4929-ad88-e7ee2258e13d, kind: Service { id: 0a835148-a2f0-40d4-bc66-05f1d779465b }, name: Name(\"nexus-0a835148-a2f0-40d4-bc66-05f1d779465b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 72])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2bea071a-6ee2-45ba-9a69-93a0e2e5b31b, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 8c296dc4-9b38-40c8-9549-80b2967f6cdc, kind: Service { id: 2bea071a-6ee2-45ba-9a69-93a0e2e5b31b }, name: Name(\"nexus-2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8dc5039d-cb88-49dd-8ab8-2ca47e481b18, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: e79fbd89-0714-4c5d-b550-de2def94225f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 16f9d651-64fc-4e3c-a69b-bda36e263e78, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: afeebd42-9418-4a17-adf1-4ce6f3cadd20, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 5eefa476-ecad-43d0-a297-e34591893cf5, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a18cc15b-758a-467a-8a3b-ed115f6e1135, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 36c43d98-b274-450d-98d0-db7e0ee7514c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 9227e547-c76f-4bc5-b6f2-101d008d915e, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 608a3190-2550-47a2-a23f-81797b571827, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: fca244c4-2682-42f7-9297-53c69ba4caa1, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:00.086076424Z","hostname":"buskin","pid":1661,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6507a4bd-9097-45d0-9787-1240feff9525, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:00.086972911Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6507a4bd-9097-45d0-9787-1240feff9525, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:00.086990362Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:00.110182282Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:00.110259104Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:00.131074143Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:00.131133695Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 416eb39a-1cea-4752-bc22-195679c44853, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:00.150587943Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 416eb39a-1cea-4752-bc22-195679c44853, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:00.150644554Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bdd869e5-e908-42b7-afb7-35517ea5ca46, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:00.169676969Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bdd869e5-e908-42b7-afb7-35517ea5ca46, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:00.169739321Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd70b00d-6277-40dc-9a7a-524663f1a340, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:00.187814457Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bd70b00d-6277-40dc-9a7a-524663f1a340, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:00.187874509Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:00.206745259Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:00.20679282Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 777ca252-0e2b-4952-8163-fd713097cfd8, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:00.225172525Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 777ca252-0e2b-4952-8163-fd713097cfd8, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:00.225228377Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:00.242680654Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:00.242750216Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:00.261582295Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:00.261650897Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:00.281592249Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:00.281646141Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:00.300358896Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:00.300406157Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5eefa476-ecad-43d0-a297-e34591893cf5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:00.318592197Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5eefa476-ecad-43d0-a297-e34591893cf5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:00.318652219Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a18cc15b-758a-467a-8a3b-ed115f6e1135, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:00.337177018Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a18cc15b-758a-467a-8a3b-ed115f6e1135, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:00.33723008Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 36c43d98-b274-450d-98d0-db7e0ee7514c, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:00.355165701Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 36c43d98-b274-450d-98d0-db7e0ee7514c, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:00.355214273Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9227e547-c76f-4bc5-b6f2-101d008d915e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:00.374041231Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9227e547-c76f-4bc5-b6f2-101d008d915e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:00.374090253Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:00.393264202Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:00.393318074Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 608a3190-2550-47a2-a23f-81797b571827, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:00.411796672Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 608a3190-2550-47a2-a23f-81797b571827, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:00.411848753Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:00.430179767Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:00.430231749Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fca244c4-2682-42f7-9297-53c69ba4caa1, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:00.448842231Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fca244c4-2682-42f7-9297-53c69ba4caa1, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:00.448903782Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:00.467183334Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:00.467247936Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:00.467473643Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:00.467630928Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:00.562218855Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:00.562273177Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:00.562281067Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:00.562285737Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:00.562290117Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:00.562295637Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:00.562299857Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:00.562304257Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:00.562327258Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:00.562335268Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:00.562340719Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:00.562347159Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:00.562353619Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:00.562358779Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:00.562370559Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:00.562375109Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:00.56238044Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:00.56238481Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:00.56238904Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:00.562486143Z","hostname":"buskin","pid":1661,"uri":"/omicron-zones","method":"PUT","req_id":"9e858ccb-9cbb-4e12-8625-16156c5c0149","remote_addr":"[fd00:1122:3344:101::1]:58669","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18\", TimeNotSynchronized), (\"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a\", TimeNotSynchronized), (\"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c\", TimeNotSynchronized), (\"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78\", TimeNotSynchronized), (\"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\", TimeNotSynchronized), (\"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b\", TimeNotSynchronized), (\"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\", TimeNotSynchronized), (\"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20\", TimeNotSynchronized), (\"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a\", TimeNotSynchronized), (\"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b\", TimeNotSynchronized), (\"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4\", TimeNotSynchronized), (\"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5\", TimeNotSynchronized), (\"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135\", TimeNotSynchronized), (\"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e\", TimeNotSynchronized), (\"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b\", TimeNotSynchronized), (\"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827\", TimeNotSynchronized), (\"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1\", TimeNotSynchronized), (\"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71\", TimeNotSynchronized), (\"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f\", TimeNotSynchronized)]","latency_us":475888,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:00.562695359Z","hostname":"buskin","pid":1661,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"8.907588775s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9e858ccb-9cbb-4e12-8625-16156c5c0149\", \"content-length\": \"124\", \"date\": \"Thu, 07 Mar 2024 22:46:00 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"9e858ccb-9cbb-4e12-8625-16156c5c0149\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 6507a4bd-9097-45d0-9787-1240feff9525, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 65847932-72ba-40f7-b62c-5adce2529400, kind: Service { id: f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf }, name: Name(\"ntp-f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 416eb39a-1cea-4752-bc22-195679c44853, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bdd869e5-e908-42b7-afb7-35517ea5ca46, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bd70b00d-6277-40dc-9a7a-524663f1a340, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 777ca252-0e2b-4952-8163-fd713097cfd8, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 0b3b3eae-a95f-4091-984a-93fccf167e79, kind: Service { id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c }, name: Name(\"external-dns-e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: ca9c5211-aa62-4394-a496-e20bf51739bc, kind: Service { id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b }, name: Name(\"external-dns-472c8e8c-ac8f-4193-921c-49dd3ceb288b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 761d7d83-4f44-4dad-ba8f-694aeae46e71, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 1eeeae19-992a-47f8-9e7b-dc93c05f0958, kind: Service { id: 761d7d83-4f44-4dad-ba8f-694aeae46e71 }, name: Name(\"nexus-761d7d83-4f44-4dad-ba8f-694aeae46e71\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a835148-a2f0-40d4-bc66-05f1d779465b, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 89eadaf9-099e-4929-ad88-e7ee2258e13d, kind: Service { id: 0a835148-a2f0-40d4-bc66-05f1d779465b }, name: Name(\"nexus-0a835148-a2f0-40d4-bc66-05f1d779465b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 72])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2bea071a-6ee2-45ba-9a69-93a0e2e5b31b, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 8c296dc4-9b38-40c8-9549-80b2967f6cdc, kind: Service { id: 2bea071a-6ee2-45ba-9a69-93a0e2e5b31b }, name: Name(\"nexus-2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8dc5039d-cb88-49dd-8ab8-2ca47e481b18, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: e79fbd89-0714-4c5d-b550-de2def94225f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 16f9d651-64fc-4e3c-a69b-bda36e263e78, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: afeebd42-9418-4a17-adf1-4ce6f3cadd20, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 5eefa476-ecad-43d0-a297-e34591893cf5, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a18cc15b-758a-467a-8a3b-ed115f6e1135, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 36c43d98-b274-450d-98d0-db7e0ee7514c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 9227e547-c76f-4bc5-b6f2-101d008d915e, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 608a3190-2550-47a2-a23f-81797b571827, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: fca244c4-2682-42f7-9297-53c69ba4caa1, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:09.471889434Z","hostname":"buskin","pid":1661,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6507a4bd-9097-45d0-9787-1240feff9525, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:09.472564825Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6507a4bd-9097-45d0-9787-1240feff9525, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:09.472587636Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:09.495328982Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:09.495374314Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:09.515012077Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:09.515052698Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 416eb39a-1cea-4752-bc22-195679c44853, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:09.535046732Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 416eb39a-1cea-4752-bc22-195679c44853, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:09.535085533Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bdd869e5-e908-42b7-afb7-35517ea5ca46, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:09.553544811Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bdd869e5-e908-42b7-afb7-35517ea5ca46, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:09.553581642Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd70b00d-6277-40dc-9a7a-524663f1a340, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:09.572029319Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bd70b00d-6277-40dc-9a7a-524663f1a340, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:09.5720683Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:09.589934539Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:09.58997551Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 777ca252-0e2b-4952-8163-fd713097cfd8, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:09.607778738Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 777ca252-0e2b-4952-8163-fd713097cfd8, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:09.607816519Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:09.625143622Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:09.625180503Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:09.641175997Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:09.641213358Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:09.65883551Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:09.658872331Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:09.676078661Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:09.676132762Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5eefa476-ecad-43d0-a297-e34591893cf5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:09.693638031Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5eefa476-ecad-43d0-a297-e34591893cf5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:09.693675362Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a18cc15b-758a-467a-8a3b-ed115f6e1135, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:09.712044077Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a18cc15b-758a-467a-8a3b-ed115f6e1135, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:09.712089368Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 36c43d98-b274-450d-98d0-db7e0ee7514c, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:09.729883376Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 36c43d98-b274-450d-98d0-db7e0ee7514c, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:09.729921987Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9227e547-c76f-4bc5-b6f2-101d008d915e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:09.748450297Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9227e547-c76f-4bc5-b6f2-101d008d915e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:09.748489768Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:09.766729399Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:09.76676674Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 608a3190-2550-47a2-a23f-81797b571827, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:09.785112004Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 608a3190-2550-47a2-a23f-81797b571827, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:09.785150365Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:09.802744226Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:09.802781518Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fca244c4-2682-42f7-9297-53c69ba4caa1, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:09.82008097Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fca244c4-2682-42f7-9297-53c69ba4caa1, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:09.820132182Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:09.837516177Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:09.837566898Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:09.837802455Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:09.83795806Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:09.932291129Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:09.932357281Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:09.932365631Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:09.932371101Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:09.932376352Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:09.932410233Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:09.932417633Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:09.932425953Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:09.932434454Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:09.932442974Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:09.932451414Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:09.932458224Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:09.932466674Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:09.932475035Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:09.932486915Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:09.932492215Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:09.932497235Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:09.932502116Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:09.932513776Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:09.932578308Z","hostname":"buskin","pid":1661,"uri":"/omicron-zones","method":"PUT","req_id":"7d4b0fb6-30fc-48c8-adde-cca579e29f55","remote_addr":"[fd00:1122:3344:101::1]:58669","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4\", TimeNotSynchronized), (\"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c\", TimeNotSynchronized), (\"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b\", TimeNotSynchronized), (\"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1\", TimeNotSynchronized), (\"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\", TimeNotSynchronized), (\"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78\", TimeNotSynchronized), (\"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e\", TimeNotSynchronized), (\"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\", TimeNotSynchronized), (\"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5\", TimeNotSynchronized), (\"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f\", TimeNotSynchronized), (\"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827\", TimeNotSynchronized), (\"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18\", TimeNotSynchronized), (\"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135\", TimeNotSynchronized), (\"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b\", TimeNotSynchronized), (\"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a\", TimeNotSynchronized), (\"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a\", TimeNotSynchronized), (\"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b\", TimeNotSynchronized), (\"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71\", TimeNotSynchronized), (\"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20\", TimeNotSynchronized)]","latency_us":460225,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:09.932811225Z","hostname":"buskin","pid":1661,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"9.182711966s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7d4b0fb6-30fc-48c8-adde-cca579e29f55\", \"content-length\": \"124\", \"date\": \"Thu, 07 Mar 2024 22:46:09 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"7d4b0fb6-30fc-48c8-adde-cca579e29f55\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:13.153376583Z","hostname":"buskin","pid":1661,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:41176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:13.153483066Z","hostname":"buskin","pid":1661,"uri":"/baseboard","method":"GET","req_id":"e0fefbb7-d5d4-485c-9b57-00b629baf735","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:41176","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":59,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 6507a4bd-9097-45d0-9787-1240feff9525, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 65847932-72ba-40f7-b62c-5adce2529400, kind: Service { id: f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf }, name: Name(\"ntp-f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 416eb39a-1cea-4752-bc22-195679c44853, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bdd869e5-e908-42b7-afb7-35517ea5ca46, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bd70b00d-6277-40dc-9a7a-524663f1a340, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 777ca252-0e2b-4952-8163-fd713097cfd8, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 0b3b3eae-a95f-4091-984a-93fccf167e79, kind: Service { id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c }, name: Name(\"external-dns-e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: ca9c5211-aa62-4394-a496-e20bf51739bc, kind: Service { id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b }, name: Name(\"external-dns-472c8e8c-ac8f-4193-921c-49dd3ceb288b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 761d7d83-4f44-4dad-ba8f-694aeae46e71, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 1eeeae19-992a-47f8-9e7b-dc93c05f0958, kind: Service { id: 761d7d83-4f44-4dad-ba8f-694aeae46e71 }, name: Name(\"nexus-761d7d83-4f44-4dad-ba8f-694aeae46e71\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a835148-a2f0-40d4-bc66-05f1d779465b, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 89eadaf9-099e-4929-ad88-e7ee2258e13d, kind: Service { id: 0a835148-a2f0-40d4-bc66-05f1d779465b }, name: Name(\"nexus-0a835148-a2f0-40d4-bc66-05f1d779465b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 72])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2bea071a-6ee2-45ba-9a69-93a0e2e5b31b, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 8c296dc4-9b38-40c8-9549-80b2967f6cdc, kind: Service { id: 2bea071a-6ee2-45ba-9a69-93a0e2e5b31b }, name: Name(\"nexus-2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8dc5039d-cb88-49dd-8ab8-2ca47e481b18, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: e79fbd89-0714-4c5d-b550-de2def94225f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 16f9d651-64fc-4e3c-a69b-bda36e263e78, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: afeebd42-9418-4a17-adf1-4ce6f3cadd20, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 5eefa476-ecad-43d0-a297-e34591893cf5, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a18cc15b-758a-467a-8a3b-ed115f6e1135, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 36c43d98-b274-450d-98d0-db7e0ee7514c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 9227e547-c76f-4bc5-b6f2-101d008d915e, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 608a3190-2550-47a2-a23f-81797b571827, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: fca244c4-2682-42f7-9297-53c69ba4caa1, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:19.116937864Z","hostname":"buskin","pid":1661,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6507a4bd-9097-45d0-9787-1240feff9525, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:19.117723088Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6507a4bd-9097-45d0-9787-1240feff9525, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:19.117741688Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:19.140770774Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:19.140834316Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:19.160442338Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:19.16050127Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 416eb39a-1cea-4752-bc22-195679c44853, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:19.180572956Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 416eb39a-1cea-4752-bc22-195679c44853, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:19.180628877Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bdd869e5-e908-42b7-afb7-35517ea5ca46, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:19.199170948Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bdd869e5-e908-42b7-afb7-35517ea5ca46, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:19.199220449Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd70b00d-6277-40dc-9a7a-524663f1a340, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:19.217407458Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bd70b00d-6277-40dc-9a7a-524663f1a340, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:19.21746646Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:19.235402632Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:19.237860056Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 777ca252-0e2b-4952-8163-fd713097cfd8, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:19.253992363Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 777ca252-0e2b-4952-8163-fd713097cfd8, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:19.254044105Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:19.272018338Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:19.272069889Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:19.290913339Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:19.29096508Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:19.308754157Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:19.308799029Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:19.327974718Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:19.328024529Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5eefa476-ecad-43d0-a297-e34591893cf5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:19.346473197Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5eefa476-ecad-43d0-a297-e34591893cf5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:19.346531028Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a18cc15b-758a-467a-8a3b-ed115f6e1135, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:19.364980165Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a18cc15b-758a-467a-8a3b-ed115f6e1135, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:19.367473181Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 36c43d98-b274-450d-98d0-db7e0ee7514c, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:19.383630269Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 36c43d98-b274-450d-98d0-db7e0ee7514c, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:19.38368558Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9227e547-c76f-4bc5-b6f2-101d008d915e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:19.401618732Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9227e547-c76f-4bc5-b6f2-101d008d915e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:19.401676154Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:19.420179343Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:19.420236324Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 608a3190-2550-47a2-a23f-81797b571827, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:19.438849267Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 608a3190-2550-47a2-a23f-81797b571827, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:19.438913689Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:19.456902592Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:19.456951283Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fca244c4-2682-42f7-9297-53c69ba4caa1, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:19.475538135Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fca244c4-2682-42f7-9297-53c69ba4caa1, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:19.475592916Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:19.494549519Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:19.494617801Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:19.494803946Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:19.494952081Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:19.588586978Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:19.588684521Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:19.588698572Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:19.588710542Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:19.588719962Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:19.588728433Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:19.588734663Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:19.588740993Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:19.588759143Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:19.588769334Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:19.588777294Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:19.588783244Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:19.588791495Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:19.588799385Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:19.588805605Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:19.588822985Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:19.588829206Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:19.588835356Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:19.588841226Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2912","zone":"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:19.588849076Z","hostname":"buskin","pid":1661,"uri":"/omicron-zones","method":"PUT","req_id":"e82507a2-c526-483a-96f7-616efc5552bd","remote_addr":"[fd00:1122:3344:101::1]:58669","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18\", TimeNotSynchronized), (\"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4\", TimeNotSynchronized), (\"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135\", TimeNotSynchronized), (\"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827\", TimeNotSynchronized), (\"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e\", TimeNotSynchronized), (\"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a\", TimeNotSynchronized), (\"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20\", TimeNotSynchronized), (\"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b\", TimeNotSynchronized), (\"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1\", TimeNotSynchronized), (\"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b\", TimeNotSynchronized), (\"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\", TimeNotSynchronized), (\"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a\", TimeNotSynchronized), (\"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f\", TimeNotSynchronized), (\"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78\", TimeNotSynchronized), (\"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5\", TimeNotSynchronized), (\"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c\", TimeNotSynchronized), (\"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b\", TimeNotSynchronized), (\"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71\", TimeNotSynchronized), (\"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\", TimeNotSynchronized)]","latency_us":471320,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:19.589069093Z","hostname":"buskin","pid":1661,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"12.552835574s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e82507a2-c526-483a-96f7-616efc5552bd\", \"content-length\": \"124\", \"date\": \"Thu, 07 Mar 2024 22:46:19 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e82507a2-c526-483a-96f7-616efc5552bd\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 6507a4bd-9097-45d0-9787-1240feff9525, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 65847932-72ba-40f7-b62c-5adce2529400, kind: Service { id: f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf }, name: Name(\"ntp-f31f8a1a-d9b2-4f1b-9049-a0d6cec43dbf\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 105])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 416eb39a-1cea-4752-bc22-195679c44853, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bdd869e5-e908-42b7-afb7-35517ea5ca46, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bd70b00d-6277-40dc-9a7a-524663f1a340, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 777ca252-0e2b-4952-8163-fd713097cfd8, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 0b3b3eae-a95f-4091-984a-93fccf167e79, kind: Service { id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c }, name: Name(\"external-dns-e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: ca9c5211-aa62-4394-a496-e20bf51739bc, kind: Service { id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b }, name: Name(\"external-dns-472c8e8c-ac8f-4193-921c-49dd3ceb288b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 761d7d83-4f44-4dad-ba8f-694aeae46e71, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 1eeeae19-992a-47f8-9e7b-dc93c05f0958, kind: Service { id: 761d7d83-4f44-4dad-ba8f-694aeae46e71 }, name: Name(\"nexus-761d7d83-4f44-4dad-ba8f-694aeae46e71\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0a835148-a2f0-40d4-bc66-05f1d779465b, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 89eadaf9-099e-4929-ad88-e7ee2258e13d, kind: Service { id: 0a835148-a2f0-40d4-bc66-05f1d779465b }, name: Name(\"nexus-0a835148-a2f0-40d4-bc66-05f1d779465b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 72])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2bea071a-6ee2-45ba-9a69-93a0e2e5b31b, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 8c296dc4-9b38-40c8-9549-80b2967f6cdc, kind: Service { id: 2bea071a-6ee2-45ba-9a69-93a0e2e5b31b }, name: Name(\"nexus-2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8dc5039d-cb88-49dd-8ab8-2ca47e481b18, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: e79fbd89-0714-4c5d-b550-de2def94225f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 16f9d651-64fc-4e3c-a69b-bda36e263e78, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: afeebd42-9418-4a17-adf1-4ce6f3cadd20, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 5eefa476-ecad-43d0-a297-e34591893cf5, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a18cc15b-758a-467a-8a3b-ed115f6e1135, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 36c43d98-b274-450d-98d0-db7e0ee7514c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 9227e547-c76f-4bc5-b6f2-101d008d915e, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 608a3190-2550-47a2-a23f-81797b571827, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: fca244c4-2682-42f7-9297-53c69ba4caa1, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:32.136499837Z","hostname":"buskin","pid":1661,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6507a4bd-9097-45d0-9787-1240feff9525, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:32.137287231Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6507a4bd-9097-45d0-9787-1240feff9525, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:32.137306891Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:32.16135776Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 03f2cdde-72ed-4fb6-bd9a-4f874ad9e4f3, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:32.161415392Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:32.184657685Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6e31b68c-68c6-4db7-adbb-2ed9a659e17e, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:32.184719077Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 416eb39a-1cea-4752-bc22-195679c44853, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:32.20495175Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 416eb39a-1cea-4752-bc22-195679c44853, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:32.205007642Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bdd869e5-e908-42b7-afb7-35517ea5ca46, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:32.224402019Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bdd869e5-e908-42b7-afb7-35517ea5ca46, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:32.224457071Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd70b00d-6277-40dc-9a7a-524663f1a340, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:32.242394244Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bd70b00d-6277-40dc-9a7a-524663f1a340, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:32.242460576Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:32.260583205Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1ef3fcc1-a4d4-4ae7-b002-8ec05b51492b, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:32.260640707Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 777ca252-0e2b-4952-8163-fd713097cfd8, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:32.279171078Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 777ca252-0e2b-4952-8163-fd713097cfd8, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:32.279227289Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:32.297007848Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:32.297052019Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:32.315763776Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:32.315818748Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:32.335110962Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:32.335158193Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:32.353412316Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:32.353463828Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5eefa476-ecad-43d0-a297-e34591893cf5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:32.372234436Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5eefa476-ecad-43d0-a297-e34591893cf5, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:32.372285588Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a18cc15b-758a-467a-8a3b-ed115f6e1135, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:32.390612203Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a18cc15b-758a-467a-8a3b-ed115f6e1135, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:32.390668104Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 36c43d98-b274-450d-98d0-db7e0ee7514c, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:32.408813054Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 36c43d98-b274-450d-98d0-db7e0ee7514c, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:32.408863955Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9227e547-c76f-4bc5-b6f2-101d008d915e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:32.427692106Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9227e547-c76f-4bc5-b6f2-101d008d915e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:32.427745767Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:32.446833936Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0c085ad2-e5e8-4683-9fed-dd84bc890e5b, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:32.446889967Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 608a3190-2550-47a2-a23f-81797b571827, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:32.465542072Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 608a3190-2550-47a2-a23f-81797b571827, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:32.465594884Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:32.483756594Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9a6f2c90-9df5-4d09-be49-6a0e3387814a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:32.483807585Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fca244c4-2682-42f7-9297-53c69ba4caa1, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:32.502014347Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fca244c4-2682-42f7-9297-53c69ba4caa1, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:32.502069178Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:32.520499376Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:32.520585919Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:32.520823116Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:32.520972631Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:32.613072159Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:254"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:32.614952087Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:254"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:32.627559178Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:254"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:32.627853827Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:32.639210721Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:254"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:32.677895363Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:254"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:32.678120309Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:32.67814858Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:32.678169941Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:32.678185611Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:32.69003644Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:32.878792027Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:32.87887644Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:32.87888455Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:32.87889068Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:32.87890472Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:32.87891155Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:32.878917621Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:32.878923001Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:32.879448957Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:32.885382086Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:32.890598214Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:32.894871564Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:32.899889896Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:32.904500196Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.058651834Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.063102729Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.067226434Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.071526004Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.075444673Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.080144125Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Configuring new Omicron zone: oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.085306921Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.090418606Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.095331385Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.101259495Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.106617367Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.112088955Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.117546578Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_608a3190-2550-47a2-a23f-81797b571827","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.123282611Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.129776988Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.136613125Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.142425881Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.150371762Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.156742235Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.162010114Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.167091968Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_608a3190-2550-47a2-a23f-81797b571827","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.17245568Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.218877077Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:271","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.262648242Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:271","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.299618852Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:271","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.335086286Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:271","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.368555239Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:271","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.370461327Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:296"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.370477798Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:188","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.425058961Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte1\", ip: 172.30.2.6, mac: MacAddr6([168, 64, 37, 255, 202, 72]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte1\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.425115923Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:1269","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.427250959Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:296"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.427268748Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:188","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.477818779Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte2\", ip: 172.30.2.7, mac: MacAddr6([168, 64, 37, 255, 234, 48]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte2\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.47787144Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:1269","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.479654774Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:296"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.479678315Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:188","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.531760482Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte3\", ip: 172.30.1.6, mac: MacAddr6([168, 64, 37, 255, 183, 87]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 }, vnic: \"vopte3\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.531811704Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:1269","zone_type":"external_dns"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.533954887Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:296"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.53398398Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:188","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.584089716Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte4\", ip: 172.30.1.5, mac: MacAddr6([168, 64, 37, 255, 214, 154]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 }, vnic: \"vopte4\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.584149198Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:1269","zone_type":"external_dns"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.586081567Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:296"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.586106207Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:188","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.634633627Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte5\", ip: 172.30.2.5, mac: MacAddr6([168, 64, 37, 255, 213, 172]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte5\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.634682441Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:1269","zone_type":"nexus"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.1.167,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.635852634Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:98"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.1.167,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.635869265Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:105"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.1.166,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.636132502Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:98"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.1.166,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.636144583Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:105"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.1.168,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.636351659Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:98"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.1.168,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.636359049Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:105"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.1.165,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.636689334Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:98"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.1.165,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.63669896Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:105"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.1.164,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.636762922Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:98"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.1.164,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.636773502Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:105"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.1.166,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.636855744Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:133"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.638868855Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.1.167,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.638900266Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:133"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.1.168,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.642551187Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:133"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.1.165,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.644395193Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:133"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.644408763Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.644417713Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.1.164,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.646479316Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:133"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.648573119Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.64857963Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Configuring new Omicron zone: oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.783369472Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.788907279Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.795321024Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.801370877Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.806985977Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.812250056Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.817528226Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.824136626Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.830424847Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:33.837228083Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:43.417151248Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:43.472424712Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:43.550836626Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:43.553121255Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:43.701445618Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:43.729555649Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:43.791036961Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:43.798098875Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:43.802869829Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:43.809367216Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:43.814728508Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:43.817049799Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:43.821771221Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:43.825972979Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_608a3190-2550-47a2-a23f-81797b571827:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:43.829794334Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:43.831020132Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:43.856861524Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:43.867900669Z","hostname":"buskin","pid":1661,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:33546"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:43.872553909Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:43.906338083Z","hostname":"buskin","pid":1661,"uri":"/baseboard","method":"GET","req_id":"8e925e13-2490-494e-96ac-290c92397a39","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:33546","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":36547,"response_code":"200"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:43.908922411Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:43.917722197Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:43.943955172Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:43.968225607Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:43.988156481Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:44.104096753Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:44.232832582Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:44.23312235Z","hostname":"buskin","pid":1661,"zone":"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4\") failed: Property not found. retry in 51.437107ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:44.517562432Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f\") failed: Property not found. retry in 59.260021ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:44.529477161Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20\") failed: Property not found. retry in 42.652418ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:44.540821884Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a\") failed: Property not found. retry in 62.906922ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:44.552473066Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c\") failed: Property not found. retry in 38.119102ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:44.562721565Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18\") failed: Property not found. retry in 50.971659ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:44.572336066Z","hostname":"buskin","pid":1661,"zone":"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78\") failed: Property not found. retry in 66.255464ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:44.581999667Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e\") failed: Property not found. retry in 59.127299ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:44.591276738Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b\") failed: Property not found. retry in 31.271765ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:44.601277969Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1\") failed: Property not found. retry in 37.419222ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:44.611195929Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5\") failed: Property not found. retry in 25.514709ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:44.686221845Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135\") failed: Property not found. retry in 59.692931ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:44.902890489Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827\") failed: Property not found. retry in 37.193493ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:44.972312616Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f\") failed: Property not found. retry in 128.92607ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:45.076832033Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20\") failed: Property not found. retry in 76.7169ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:45.119308696Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4\") failed: Property not found. retry in 94.183619ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:45.147751575Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a\") failed: Property not found. retry in 113.669325ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:45.188278739Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c\") failed: Property not found. retry in 137.283337ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:45.233902937Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18\") failed: Property not found. retry in 110.612783ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:45.285090743Z","hostname":"buskin","pid":1661,"zone":"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78\") failed: Property not found. retry in 66.93752ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:45.332361031Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e\") failed: Property not found. retry in 76.079785ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:45.359907413Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b\") failed: Property not found. retry in 50.543313ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:45.386933589Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1\") failed: Property not found. retry in 122.235474ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:45.42006534Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5\") failed: Property not found. retry in 88.878608ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:45.440886579Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135\") failed: Property not found. retry in 134.562677ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:45.45682054Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827\") failed: Property not found. retry in 61.75429ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:45.471002628Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f\") failed: Property not found. retry in 101.247461ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:45.487028102Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20\") failed: Property not found. retry in 193.144747ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:45.502287753Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4\") failed: Property not found. retry in 139.898985ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:45.517789371Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a\") failed: Property not found. retry in 279.895626ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:45.534493266Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c\") failed: Property not found. retry in 269.450152ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:45.550258721Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18\") failed: Property not found. retry in 191.012803ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:45.566346647Z","hostname":"buskin","pid":1661,"zone":"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78\") failed: Property not found. retry in 294.036579ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:45.579917297Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e\") failed: Property not found. retry in 118.545929ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:45.593867498Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b\") failed: Property not found. retry in 197.924376ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:45.607727527Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1\") failed: Property not found. retry in 165.533565ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:45.620859923Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5\") failed: Property not found. retry in 164.123161ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:45.634430533Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135\") failed: Property not found. retry in 204.412221ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:45.646504188Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827\") failed: Property not found. retry in 153.919528ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:45.659951304Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f\") failed: Property not found. retry in 376.401671ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:45.672826133Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4\") failed: Property not found. retry in 200.009515ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:45.684078543Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20\") failed: Property not found. retry in 463.073083ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:45.707676656Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e\") failed: Property not found. retry in 552.296339ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:45.724597617Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:45.73895132Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:45.740143266Z","hostname":"buskin","pid":1661,"zone":"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:45.750149539Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:45.750722386Z","hostname":"buskin","pid":1661,"zone":"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18\") failed: Property not found. retry in 386.206334ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:45.772760701Z","hostname":"buskin","pid":1661,"zone":"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1\") failed: Property not found. retry in 585.165887ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:45.799139798Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5\") failed: Property not found. retry in 356.329483ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:45.810398958Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b\") failed: Property not found. retry in 348.512087ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:45.824793673Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827\") failed: Property not found. retry in 204.696794ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:45.853146459Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a\") failed: Property not found. retry in 383.792275ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:45.877633569Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c\") failed: Property not found. retry in 223.108242ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:45.896722135Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78\") failed: Property not found. retry in 417.831138ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:45.916055839Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135\") failed: Property not found. retry in 527.360446ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:45.935969171Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4\") failed: Property not found. retry in 543.204272ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:45.953052967Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f\") failed: Property not found. retry in 857.167973ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:46.071198345Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b\") failed: Property not found. retry in 65.354495ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:46.088629502Z","hostname":"buskin","pid":1661,"zone":"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\") failed: Property not found. retry in 41.651392ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:46.10445967Z","hostname":"buskin","pid":1661,"zone":"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827\") failed: Property not found. retry in 733.322142ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:46.123031721Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c\") failed: Property not found. retry in 553.407944ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:46.142480048Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\") failed: Property not found. retry in 114.826148ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:46.163279736Z","hostname":"buskin","pid":1661,"zone":"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18\") failed: Property not found. retry in 556.374763ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:46.244010235Z","hostname":"buskin","pid":1661,"zone":"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b\") failed: Property not found. retry in 501.446811ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:46.299829421Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5\") failed: Property not found. retry in 670.418895ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:46.319707771Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b\") failed: Property not found. retry in 80.501933ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:46.333337293Z","hostname":"buskin","pid":1661,"zone":"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\") failed: Property not found. retry in 170.248287ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:46.346225992Z","hostname":"buskin","pid":1661,"zone":"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20\") failed: Property not found. retry in 1.00209874s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:46.362548445Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a\") failed: Property not found. retry in 976.56408ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:46.38290436Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78\") failed: Property not found. retry in 624.74871ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:46.401606064Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e\") failed: Property not found. retry in 1.199577969s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:46.419139204Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1\") failed: Property not found. retry in 983.34785ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:46.437677514Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b\") failed: Property not found. retry in 189.017185ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:46.453365108Z","hostname":"buskin","pid":1661,"zone":"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135\") failed: Property not found. retry in 698.076153ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:46.483007453Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4\") failed: Property not found. retry in 557.855107ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:46.513188055Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\") failed: Property not found. retry in 427.810216ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:46.535989263Z","hostname":"buskin","pid":1661,"zone":"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b\") failed: Property not found. retry in 322.417814ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:46.658313538Z","hostname":"buskin","pid":1661,"zone":"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c\") failed: Property not found. retry in 855.895599ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:46.712769762Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18\") failed: Property not found. retry in 1.42569256s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:46.819950459Z","hostname":"buskin","pid":1661,"zone":"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b\") failed: Property not found. retry in 1.341524594s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:46.835958553Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827\") failed: Property not found. retry in 993.038461ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:46.875796166Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f\") failed: Property not found. retry in 1.449818437s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:46.945748829Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\") failed: Property not found. retry in 500.134836ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:46.981178089Z","hostname":"buskin","pid":1661,"zone":"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b\") failed: Property not found. retry in 927.95901ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:47.00569512Z","hostname":"buskin","pid":1661,"zone":"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5\") failed: Property not found. retry in 1.329628554s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:47.030983804Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78\") failed: Property not found. retry in 1.277981516s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:47.053799513Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:47.055294328Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:47.055762442Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:47.056015Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:47.056150573Z","hostname":"buskin","pid":1661,"zone":"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:47.064936379Z","hostname":"buskin","pid":1661,"zone":"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:47.073329663Z","hostname":"buskin","pid":1661,"zone":"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4\") failed: Property not found. retry in 1.087859121s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:47.099500963Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135\") failed: Property not found. retry in 573.500434ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:47.202765592Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a\") failed: Property not found. retry in 617.306537ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:47.380730787Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\") failed: Property not found. retry in 41.707371ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:47.395804292Z","hostname":"buskin","pid":1661,"zone":"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20\") failed: Property not found. retry in 1.029041139s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:47.412722273Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71\") failed: Property not found. retry in 27.19789ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:47.425755117Z","hostname":"buskin","pid":1661,"zone":"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1\") failed: Property not found. retry in 1.405344124s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:47.442855223Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b\") failed: Property not found. retry in 74.516822ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:47.458431264Z","hostname":"buskin","pid":1661,"zone":"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\") failed: Property not found. retry in 57.558582ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:47.567532808Z","hostname":"buskin","pid":1661,"zone":"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\") failed: Property not found. retry in 578.87427ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:47.734050898Z","hostname":"buskin","pid":1661,"zone":"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71\") failed: Property not found. retry in 81.094857ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:47.771277532Z","hostname":"buskin","pid":1661,"zone":"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c\") failed: Property not found. retry in 1.305062467s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:47.796499404Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e\") failed: Property not found. retry in 563.645212ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:47.83511588Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135\") failed: Property not found. retry in 1.488984516s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:47.862047014Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b\") failed: Property not found. retry in 58.704334ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:47.877941014Z","hostname":"buskin","pid":1661,"zone":"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\") failed: Property not found. retry in 215.182892ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:47.889019948Z","hostname":"buskin","pid":1661,"zone":"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71\") failed: Property not found. retry in 288.037369ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:47.90066649Z","hostname":"buskin","pid":1661,"zone":"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827\") failed: Property not found. retry in 773.338248ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:47.911997162Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b\") failed: Property not found. retry in 501.156839ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:47.950529346Z","hostname":"buskin","pid":1661,"zone":"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b\") failed: Property not found. retry in 163.769297ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:47.963409635Z","hostname":"buskin","pid":1661,"zone":"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a\") failed: Property not found. retry in 1.44526112s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:48.011809677Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\") failed: Property not found. retry in 269.494423ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:48.121708016Z","hostname":"buskin","pid":1661,"zone":"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b\") failed: Property not found. retry in 431.846129ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:48.14302671Z","hostname":"buskin","pid":1661,"zone":"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b\") failed: Property not found. retry in 1.043357156s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:48.190531755Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4\") failed: Property not found. retry in 640.645575ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:48.202771805Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71\") failed: Property not found. retry in 490.942832ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:48.215375765Z","hostname":"buskin","pid":1661,"zone":"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18\") failed: Property not found. retry in 946.490437ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:48.262297253Z","hostname":"buskin","pid":1661,"zone":"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\") failed: Property not found. retry in 713.859262ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:48.327402419Z","hostname":"buskin","pid":1661,"zone":"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78\") failed: Property not found. retry in 1.010656687s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:48.344780204Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5\") failed: Property not found. retry in 710.697423ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:48.422878453Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\") failed: Property not found. retry in 727.27385ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:48.435588927Z","hostname":"buskin","pid":1661,"zone":"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f\") failed: Property not found. retry in 1.133668454s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:48.44760601Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20\") failed: Property not found. retry in 567.660625ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:48.459357554Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e\") failed: Property not found. retry in 1.268561489s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:48.470281394Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b\") failed: Property not found. retry in 1.161383103s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:48.481131132Z","hostname":"buskin","pid":1661,"zone":"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b\") failed: Property not found. retry in 788.290312ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:48.589482054Z","hostname":"buskin","pid":1661,"zone":"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827\") failed: Property not found. retry in 1.158962126s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:48.698963881Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71\") failed: Property not found. retry in 622.525254ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:48.719287575Z","hostname":"buskin","pid":1661,"zone":"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4\") failed: Property not found. retry in 828.803866ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:48.857459468Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1\") failed: Property not found. retry in 1.017447306s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:48.870369228Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20\") failed: Property not found. retry in 1.212760197s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:49.042397363Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\") failed: Property not found. retry in 1.241216069s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:49.053089997Z","hostname":"buskin","pid":1661,"zone":"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c\") failed: Property not found. retry in 1.176821047s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:49.114286955Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5\") failed: Property not found. retry in 865.682512ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:49.146149577Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\") failed: Property not found. retry in 1.134884924s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:49.175570466Z","hostname":"buskin","pid":1661,"zone":"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18\") failed: Property not found. retry in 709.157316ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:49.219960217Z","hostname":"buskin","pid":1661,"zone":"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b\") failed: Property not found. retry in 567.001913ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:49.245491828Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71\") failed: Property not found. retry in 1.075951133s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:49.353208791Z","hostname":"buskin","pid":1661,"zone":"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135\") failed: Property not found. retry in 540.660639ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:49.363228834Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78\") failed: Property not found. retry in 976.921762ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:49.373154473Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b\") failed: Property not found. retry in 727.105822ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:49.388388004Z","hostname":"buskin","pid":1661,"zone":"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a\") failed: Property not found. retry in 1.216346207s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:49.467540914Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f\") failed: Property not found. retry in 699.173594ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:49.590002152Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b\") failed: Property not found. retry in 541.533711ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:49.65514787Z","hostname":"buskin","pid":1661,"zone":"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4\") failed: Property not found. retry in 901.888091ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:49.695735616Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e\") failed: Property not found. retry in 1.039625148s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:49.749040366Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b\") failed: Property not found. retry in 904.141398ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:49.823177665Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827\") failed: Property not found. retry in 1.407815612s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:49.86768912Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1\") failed: Property not found. retry in 835.560904ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:49.899120509Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135\") failed: Property not found. retry in 1.25178021s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:49.917324879Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18\") failed: Property not found. retry in 895.035001ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:49.941489298Z","hostname":"buskin","pid":1661,"zone":"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5\") failed: Property not found. retry in 1.225206312s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:50.021852156Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b\") failed: Property not found. retry in 1.295634341s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:50.126854227Z","hostname":"buskin","pid":1661,"zone":"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b\") failed: Property not found. retry in 1.363219684s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:50.207297577Z","hostname":"buskin","pid":1661,"zone":"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20\") failed: Property not found. retry in 1.140646699s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:50.264903147Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f\") failed: Property not found. retry in 876.058025ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:50.299263475Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\") failed: Property not found. retry in 1.047323986s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:50.307248846Z","hostname":"buskin","pid":1661,"zone":"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c\") failed: Property not found. retry in 1.051082259s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:50.315361721Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\") failed: Property not found. retry in 794.781425ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:50.324549078Z","hostname":"buskin","pid":1661,"zone":"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78\") failed: Property not found. retry in 1.490394927s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:50.427556359Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71\") failed: Property not found. retry in 1.189101239s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:50.441661385Z","hostname":"buskin","pid":1661,"zone":"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4\") failed: Property not found. retry in 1.274355591s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:50.608097451Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a\") failed: Property not found. retry in 818.262879ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:50.693448869Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b\") failed: Property not found. retry in 969.152055ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:50.735426507Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1\") failed: Property not found. retry in 820.150558ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:50.74645673Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e\") failed: Property not found. retry in 697.663994ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:50.79909648Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18\") failed: Property not found. retry in 525.82792ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:50.845635916Z","hostname":"buskin","pid":1661,"zone":"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\") failed: Property not found. retry in 521.033539ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:51.128358845Z","hostname":"buskin","pid":1661,"zone":"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135\") failed: Property not found. retry in 578.345501ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:51.179056056Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f\") failed: Property not found. retry in 1.439834588s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:51.187936115Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5\") failed: Property not found. retry in 1.31419859s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:51.255538166Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827\") failed: Property not found. retry in 1.432487356s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:51.284764049Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\") failed: Property not found. retry in 1.34187664s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:51.365083915Z","hostname":"buskin","pid":1661,"zone":"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c\") failed: Property not found. retry in 1.307106295s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:51.37551593Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18\") failed: Property not found. retry in 1.114241434s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:51.383324536Z","hostname":"buskin","pid":1661,"zone":"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20\") failed: Property not found. retry in 884.254364ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:51.414745965Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b\") failed: Property not found. retry in 685.423394ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:51.432304295Z","hostname":"buskin","pid":1661,"zone":"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e\") failed: Property not found. retry in 1.458736517s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:51.506320201Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a\") failed: Property not found. retry in 1.46570044s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:51.522025665Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1\") failed: Property not found. retry in 709.577665ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:51.576725817Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b\") failed: Property not found. retry in 1.441913575s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:51.58446354Z","hostname":"buskin","pid":1661,"zone":"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71\") failed: Property not found. retry in 973.720233ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:51.639991697Z","hostname":"buskin","pid":1661,"zone":"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\") failed: Property not found. retry in 1.212335076s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:51.657760924Z","hostname":"buskin","pid":1661,"zone":"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b\") failed: Property not found. retry in 803.826229ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:51.714065395Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135\") failed: Property not found. retry in 1.41120206s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:51.81654392Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:51.819149139Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:46:51.819395146Z","hostname":"buskin","pid":1661,"zone":"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4\") failed: Property not found. retry in 745.188453ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:51.891306488Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78\") failed: Property not found. retry in 970.298779ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:51.927647386Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a\") failed: Property not found. retry in 55.627164ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:52.021274133Z","hostname":"buskin","pid":1661,"zone":"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a\") failed: Property not found. retry in 111.981481ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:52.090146624Z","hostname":"buskin","pid":1661,"zone":"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b\") failed: Property not found. retry in 1.202115194s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:52.131323937Z","hostname":"buskin","pid":1661,"zone":"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a\") failed: Property not found. retry in 217.461493ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:52.213409487Z","hostname":"buskin","pid":1661,"zone":"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1\") failed: Property not found. retry in 1.253397834s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:52.295496316Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20\") failed: Property not found. retry in 875.64204ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:52.309397236Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a\") failed: Property not found. retry in 303.480532ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:52.440174246Z","hostname":"buskin","pid":1661,"zone":"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18\") failed: Property not found. retry in 769.290717ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:52.507776218Z","hostname":"buskin","pid":1661,"zone":"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b\") failed: Property not found. retry in 907.45568ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:52.528293157Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5\") failed: Property not found. retry in 1.232767636s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:52.578999498Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71\") failed: Property not found. retry in 1.05310608s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:52.623805251Z","hostname":"buskin","pid":1661,"zone":"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f\") failed: Property not found. retry in 649.936698ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:52.637561967Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4\") failed: Property not found. retry in 538.369752ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:52.645404244Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c\") failed: Property not found. retry in 957.36492ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:52.692581319Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\") failed: Property not found. retry in 704.153042ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:52.715717907Z","hostname":"buskin","pid":1661,"zone":"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827\") failed: Property not found. retry in 623.089207ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:52.725907955Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a\") failed: Property not found. retry in 1.168196879s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:52.753604002Z","hostname":"buskin","pid":1661,"zone":"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\") failed: Property not found. retry in 1.440963044s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:52.879459023Z","hostname":"buskin","pid":1661,"zone":"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78\") failed: Property not found. retry in 881.497744ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:52.907343545Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e\") failed: Property not found. retry in 1.025888501s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:52.974741061Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a\") failed: Property not found. retry in 1.232560201s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:52.997817358Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b\") failed: Property not found. retry in 1.143960873s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:53.03663004Z","hostname":"buskin","pid":1661,"zone":"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4\") failed: Property not found. retry in 509.112519ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:53.197917282Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20\") failed: Property not found. retry in 787.403186ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:53.207852691Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135\") failed: Property not found. retry in 599.997206ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:53.245842409Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18\") failed: Property not found. retry in 1.130875858s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:53.290556Z","hostname":"buskin","pid":1661,"zone":"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f\") failed: Property not found. retry in 880.90589ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:53.309406129Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b\") failed: Property not found. retry in 702.761046ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:53.344237651Z","hostname":"buskin","pid":1661,"zone":"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827\") failed: Property not found. retry in 601.216037ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:53.358997957Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\") failed: Property not found. retry in 885.830218ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:53.4309429Z","hostname":"buskin","pid":1661,"zone":"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b\") failed: Property not found. retry in 1.382878485s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:53.445889451Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1\") failed: Property not found. retry in 1.353721177s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:53.5591046Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c\") failed: Property not found. retry in 552.692923ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:53.659551294Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71\") failed: Property not found. retry in 1.302648542s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:53.685312752Z","hostname":"buskin","pid":1661,"zone":"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4\") failed: Property not found. retry in 1.027793803s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:53.715950247Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78\") failed: Property not found. retry in 1.111467332s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:53.796748177Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5\") failed: Property not found. retry in 744.082855ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:53.821521646Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135\") failed: Property not found. retry in 837.383129ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:53.853603725Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a\") failed: Property not found. retry in 1.418275129s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:53.931763605Z","hostname":"buskin","pid":1661,"zone":"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827\") failed: Property not found. retry in 806.396624ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:53.969921288Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20\") failed: Property not found. retry in 754.561291ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:54.006100111Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e\") failed: Property not found. retry in 543.298763ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:54.013974358Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b\") failed: Property not found. retry in 1.337447917s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:54.055603376Z","hostname":"buskin","pid":1661,"zone":"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b\") failed: Property not found. retry in 1.165883996s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:54.190536221Z","hostname":"buskin","pid":1661,"zone":"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f\") failed: Property not found. retry in 930.055499ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:54.199306496Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c\") failed: Property not found. retry in 711.490223ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:54.220647761Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a\") failed: Property not found. retry in 745.030799ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:54.238573862Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\") failed: Property not found. retry in 1.007743134s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:54.326347093Z","hostname":"buskin","pid":1661,"zone":"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\") failed: Property not found. retry in 1.022130428s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:54.334499169Z","hostname":"buskin","pid":1661,"zone":"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18\") failed: Property not found. retry in 1.083531999s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:54.431132898Z","hostname":"buskin","pid":1661,"zone":"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e\") failed: Property not found. retry in 796.411306ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:54.56632352Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5\") failed: Property not found. retry in 1.358079191s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:54.575006293Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135\") failed: Property not found. retry in 1.424450307s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:54.700813312Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4\") failed: Property not found. retry in 804.443968ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:54.753982508Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20\") failed: Property not found. retry in 581.163784ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:54.769768345Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827\") failed: Property not found. retry in 740.7545ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:54.785986925Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b\") failed: Property not found. retry in 830.770366ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:54.838293835Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78\") failed: Property not found. retry in 1.005051147s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:54.917744334Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1\") failed: Property not found. retry in 1.230137932s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:54.9262048Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c\") failed: Property not found. retry in 592.787733ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:54.942144041Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a\") failed: Property not found. retry in 1.122438215s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:54.993676508Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71\") failed: Property not found. retry in 722.202691ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:55.002727721Z","hostname":"buskin","pid":1661,"zone":"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f\") failed: Property not found. retry in 1.096154379s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:55.140874364Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\") failed: Property not found. retry in 1.059635243s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:55.344471803Z","hostname":"buskin","pid":1661,"zone":"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a\") failed: Property not found. retry in 1.061526979s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:55.360218859Z","hostname":"buskin","pid":1661,"zone":"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20\") failed: Property not found. retry in 1.170653148s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:55.370942173Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e\") failed: Property not found. retry in 1.13921496s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:55.380621425Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\") failed: Property not found. retry in 1.221507117s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:55.389237855Z","hostname":"buskin","pid":1661,"zone":"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b\") failed: Property not found. retry in 945.895362ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:55.399465094Z","hostname":"buskin","pid":1661,"zone":"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b\") failed: Property not found. retry in 542.658008ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:55.409346913Z","hostname":"buskin","pid":1661,"zone":"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18\") failed: Property not found. retry in 1.187668127s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:55.52546522Z","hostname":"buskin","pid":1661,"zone":"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827\") failed: Property not found. retry in 1.219074178s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:55.535696958Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c\") failed: Property not found. retry in 1.154536272s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:55.542485803Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4\") failed: Property not found. retry in 1.283256085s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:55.568958703Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b\") failed: Property not found. retry in 717.416603ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:55.678870262Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71\") failed: Property not found. retry in 661.274087ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:55.736587896Z","hostname":"buskin","pid":1661,"zone":"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78\") failed: Property not found. retry in 605.774149ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:55.932695109Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5\") failed: Property not found. retry in 618.252949ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:55.941464913Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b\") failed: Property not found. retry in 1.199732537s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:55.96219892Z","hostname":"buskin","pid":1661,"zone":"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a\") failed: Property not found. retry in 1.364166565s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:56.126616586Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135\") failed: Property not found. retry in 518.073135ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:56.135606417Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1\") failed: Property not found. retry in 1.464277748s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:56.168019816Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f\") failed: Property not found. retry in 1.311487233s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:56.246527538Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b\") failed: Property not found. retry in 1.2877967s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:56.359332174Z","hostname":"buskin","pid":1661,"zone":"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b\") failed: Property not found. retry in 896.861944ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:56.406745066Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\") failed: Property not found. retry in 722.086816ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:56.41613867Z","hostname":"buskin","pid":1661,"zone":"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71\") failed: Property not found. retry in 836.470238ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:56.425165943Z","hostname":"buskin","pid":1661,"zone":"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a\") failed: Property not found. retry in 572.115277ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:56.433793134Z","hostname":"buskin","pid":1661,"zone":"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e\") failed: Property not found. retry in 1.363697288s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:56.529501034Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78\") failed: Property not found. retry in 910.177745ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:56.550105026Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20\") failed: Property not found. retry in 837.914591ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:56.560304754Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5\") failed: Property not found. retry in 787.212912ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:56.572032178Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\") failed: Property not found. retry in 1.040078148s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:56.621150012Z","hostname":"buskin","pid":1661,"zone":"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135\") failed: Property not found. retry in 1.358724853s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:56.665844002Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c\") failed: Property not found. retry in 656.201417ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:56.70849042Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18\") failed: Property not found. retry in 524.350732ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:56.723288897Z","hostname":"buskin","pid":1661,"zone":"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827\") failed: Property not found. retry in 1.069285043s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:56.767332007Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4\") failed: Property not found. retry in 962.826425ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:56.862676697Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a\") failed: Property not found. retry in 965.999128ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:57.016442071Z","hostname":"buskin","pid":1661,"zone":"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\") failed: Property not found. retry in 574.742843ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:57.147584712Z","hostname":"buskin","pid":1661,"zone":"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b\") failed: Property not found. retry in 1.12156743s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:57.170975979Z","hostname":"buskin","pid":1661,"zone":"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18\") failed: Property not found. retry in 1.460435114s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:57.260723059Z","hostname":"buskin","pid":1661,"zone":"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71\") failed: Property not found. retry in 824.821097ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:57.272948058Z","hostname":"buskin","pid":1661,"zone":"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b\") failed: Property not found. retry in 967.193843ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:57.314605657Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5\") failed: Property not found. retry in 925.246154ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:57.369487684Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c\") failed: Property not found. retry in 542.524988ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:57.378450355Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20\") failed: Property not found. retry in 978.136888ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:57.407117841Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78\") failed: Property not found. retry in 712.446388ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:57.470613798Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a\") failed: Property not found. retry in 1.081695184s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:57.501698547Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f\") failed: Property not found. retry in 606.722478ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:57.568375281Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1\") failed: Property not found. retry in 672.409344ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:57.642904422Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b\") failed: Property not found. retry in 1.027486878s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:57.656953216Z","hostname":"buskin","pid":1661,"zone":"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\") failed: Property not found. retry in 1.009794538s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:57.671860076Z","hostname":"buskin","pid":1661,"zone":"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\") failed: Property not found. retry in 1.178792862s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:57.732131087Z","hostname":"buskin","pid":1661,"zone":"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4\") failed: Property not found. retry in 982.704126ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:57.836251671Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827\") failed: Property not found. retry in 636.876993ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:57.848570223Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e\") failed: Property not found. retry in 1.41069637s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:57.903401529Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c\") failed: Property not found. retry in 830.77965ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:57.931370894Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a\") failed: Property not found. retry in 1.132185893s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:57.993766659Z","hostname":"buskin","pid":1661,"zone":"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135\") failed: Property not found. retry in 634.521419ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:58.038788219Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71\") failed: Property not found. retry in 1.260584318s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:58.112362571Z","hostname":"buskin","pid":1661,"zone":"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f\") failed: Property not found. retry in 1.404636791s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:58.187759048Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78\") failed: Property not found. retry in 776.236291ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:58.198590535Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b\") failed: Property not found. retry in 842.606329ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:58.293412709Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b\") failed: Property not found. retry in 828.889815ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:58.303521384Z","hostname":"buskin","pid":1661,"zone":"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5\") failed: Property not found. retry in 1.13308112s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:58.313575688Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1\") failed: Property not found. retry in 731.144755ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:58.326961732Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20\") failed: Property not found. retry in 1.119176173s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:58.400230295Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827\") failed: Property not found. retry in 618.857741ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:58.499145063Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a\") failed: Property not found. retry in 533.946219ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:58.598906555Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135\") failed: Property not found. retry in 1.26309619s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:58.685842001Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\") failed: Property not found. retry in 979.425876ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:58.698231065Z","hostname":"buskin","pid":1661,"zone":"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b\") failed: Property not found. retry in 1.002485062s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:58.709263708Z","hostname":"buskin","pid":1661,"zone":"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18\") failed: Property not found. retry in 1.055224415s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:58.73282235Z","hostname":"buskin","pid":1661,"zone":"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c\") failed: Property not found. retry in 1.312936652s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:58.77488611Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4\") failed: Property not found. retry in 1.087844577s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:58.831999555Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\") failed: Property not found. retry in 1.417713379s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:58.925569812Z","hostname":"buskin","pid":1661,"zone":"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78\") failed: Property not found. retry in 826.542271ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:58.99307113Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1\") failed: Property not found. retry in 674.973882ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:59.069301943Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827\") failed: Property not found. retry in 1.275630225s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:59.132858922Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a\") failed: Property not found. retry in 1.340953087s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:59.148253247Z","hostname":"buskin","pid":1661,"zone":"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a\") failed: Property not found. retry in 1.269165134s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:59.166036054Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b\") failed: Property not found. retry in 1.11680745s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:59.180677586Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b\") failed: Property not found. retry in 1.352226404s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:59.193154963Z","hostname":"buskin","pid":1661,"zone":"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e\") failed: Property not found. retry in 757.805004ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:59.330674657Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71\") failed: Property not found. retry in 530.169846ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:59.388689399Z","hostname":"buskin","pid":1661,"zone":"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5\") failed: Property not found. retry in 875.727578ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:59.461911581Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20\") failed: Property not found. retry in 1.224271678s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:59.53541384Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f\") failed: Property not found. retry in 673.009424ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:59.605478887Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\") failed: Property not found. retry in 1.272579016s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:59.689708591Z","hostname":"buskin","pid":1661,"zone":"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b\") failed: Property not found. retry in 718.459664ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:59.724715288Z","hostname":"buskin","pid":1661,"zone":"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1\") failed: Property not found. retry in 623.76103ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:59.756017803Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18\") failed: Property not found. retry in 1.011761547s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:59.800531148Z","hostname":"buskin","pid":1661,"zone":"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78\") failed: Property not found. retry in 627.320107ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:59.834668399Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4\") failed: Property not found. retry in 889.488729ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:59.93071084Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71\") failed: Property not found. retry in 767.082236ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:59.942490775Z","hostname":"buskin","pid":1661,"zone":"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135\") failed: Property not found. retry in 1.357913981s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:46:59.96053112Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c\") failed: Property not found. retry in 1.200663446s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:00.10023945Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e\") failed: Property not found. retry in 979.448286ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:00.112053707Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f\") failed: Property not found. retry in 1.268655854s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:00.289494016Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b\") failed: Property not found. retry in 1.461495905s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:00.309577673Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5\") failed: Property not found. retry in 1.087096195s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:00.351061926Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\") failed: Property not found. retry in 1.004232374s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:00.362908783Z","hostname":"buskin","pid":1661,"zone":"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1\") failed: Property not found. retry in 823.135326ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:00.392020183Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827\") failed: Property not found. retry in 1.137030815s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:00.425417431Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a\") failed: Property not found. retry in 664.408364ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:00.446419166Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b\") failed: Property not found. retry in 894.768184ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:00.459002496Z","hostname":"buskin","pid":1661,"zone":"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78\") failed: Property not found. retry in 827.288217ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:00.47701762Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a\") failed: Property not found. retry in 1.060838202s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:00.502322384Z","hostname":"buskin","pid":1661,"zone":"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b\") failed: Property not found. retry in 521.03981ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:00.557261543Z","hostname":"buskin","pid":1661,"zone":"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71\") failed: Property not found. retry in 988.385825ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:00.722025829Z","hostname":"buskin","pid":1661,"zone":"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20\") failed: Property not found. retry in 647.986789ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:00.77106196Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18\") failed: Property not found. retry in 1.357153561s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:00.825584777Z","hostname":"buskin","pid":1661,"zone":"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4\") failed: Property not found. retry in 562.666487ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:00.838746675Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\") failed: Property not found. retry in 911.727291ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:00.978080553Z","hostname":"buskin","pid":1661,"zone":"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b\") failed: Property not found. retry in 1.107250053s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:01.093395536Z","hostname":"buskin","pid":1661,"zone":"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e\") failed: Property not found. retry in 1.06304531s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:01.106421149Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a\") failed: Property not found. retry in 827.988881ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:01.123069892Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1\") failed: Property not found. retry in 721.187721ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:01.230777655Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c\") failed: Property not found. retry in 1.468318248s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:01.312018759Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78\") failed: Property not found. retry in 1.318383588s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:01.32298574Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135\") failed: Property not found. retry in 1.437202272s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:01.332725384Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b\") failed: Property not found. retry in 796.76781ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:01.366202415Z","hostname":"buskin","pid":1661,"zone":"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\") failed: Property not found. retry in 1.437817892s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:01.378470796Z","hostname":"buskin","pid":1661,"zone":"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4\") failed: Property not found. retry in 1.412565664s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:01.415534556Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20\") failed: Property not found. retry in 1.309197111s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:01.431099646Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5\") failed: Property not found. retry in 1.445287921s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:01.451572164Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f\") failed: Property not found. retry in 855.716859ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:01.569424043Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a\") failed: Property not found. retry in 751.20094ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:01.580548079Z","hostname":"buskin","pid":1661,"zone":"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827\") failed: Property not found. retry in 1.029827939s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:01.592030926Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71\") failed: Property not found. retry in 838.664108ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:01.729394624Z","hostname":"buskin","pid":1661,"zone":"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b\") failed: Property not found. retry in 502.665433ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:01.788326394Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\") failed: Property not found. retry in 662.52707ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:01.902660797Z","hostname":"buskin","pid":1661,"zone":"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a\") failed: Property not found. retry in 637.512465ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:01.965531987Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1\") failed: Property not found. retry in 906.895012ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:01.980976043Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b\") failed: Property not found. retry in 744.82596ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:02.176359294Z","hostname":"buskin","pid":1661,"zone":"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e\") failed: Property not found. retry in 1.460697773s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:02.192807971Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18\") failed: Property not found. retry in 954.84502ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:02.206197245Z","hostname":"buskin","pid":1661,"zone":"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b\") failed: Property not found. retry in 1.132012985s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:02.221078475Z","hostname":"buskin","pid":1661,"zone":"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b\") failed: Property not found. retry in 846.34653ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:02.303757112Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a\") failed: Property not found. retry in 1.420138323s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:02.34506634Z","hostname":"buskin","pid":1661,"zone":"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f\") failed: Property not found. retry in 1.395403491s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:02.441895464Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\") failed: Property not found. retry in 1.099923596s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:02.578283613Z","hostname":"buskin","pid":1661,"zone":"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71\") failed: Property not found. retry in 605.622693ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:02.589229844Z","hostname":"buskin","pid":1661,"zone":"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a\") failed: Property not found. retry in 502.151751ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:02.617486537Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827\") failed: Property not found. retry in 1.132681709s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:02.637999977Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78\") failed: Property not found. retry in 642.43197ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:02.659742743Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20\") failed: Property not found. retry in 1.475069403s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:02.753429143Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135\") failed: Property not found. retry in 971.08414ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:02.786973016Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c\") failed: Property not found. retry in 639.466273ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:02.800128723Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\") failed: Property not found. retry in 872.006532ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:02.828601903Z","hostname":"buskin","pid":1661,"zone":"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4\") failed: Property not found. retry in 706.737741ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:02.844441982Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1\") failed: Property not found. retry in 1.116186725s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:02.89934685Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b\") failed: Property not found. retry in 814.198052ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:02.933423059Z","hostname":"buskin","pid":1661,"zone":"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a\") failed: Property not found. retry in 798.872659ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:03.132468821Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b\") failed: Property not found. retry in 593.109362ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:03.169360546Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18\") failed: Property not found. retry in 1.367325922s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:03.183026738Z","hostname":"buskin","pid":1661,"zone":"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71\") failed: Property not found. retry in 699.300935ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:03.207224499Z","hostname":"buskin","pid":1661,"zone":"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78\") failed: Property not found. retry in 616.991036ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:03.312402326Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b\") failed: Property not found. retry in 662.299927ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:03.362650304Z","hostname":"buskin","pid":1661,"zone":"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c\") failed: Property not found. retry in 1.325195761s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:03.450884818Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4\") failed: Property not found. retry in 1.457555947s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:03.561049965Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e\") failed: Property not found. retry in 833.683019ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:03.666142779Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\") failed: Property not found. retry in 1.030982752s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:03.68998618Z","hostname":"buskin","pid":1661,"zone":"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\") failed: Property not found. retry in 845.430167ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:03.710974004Z","hostname":"buskin","pid":1661,"zone":"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b\") failed: Property not found. retry in 1.068953451s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:03.757251361Z","hostname":"buskin","pid":1661,"zone":"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a\") failed: Property not found. retry in 1.442804677s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:03.778754991Z","hostname":"buskin","pid":1661,"zone":"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b\") failed: Property not found. retry in 984.513934ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:03.787498805Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827\") failed: Property not found. retry in 697.051644ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:03.796209978Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f\") failed: Property not found. retry in 1.38626966s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:03.848990242Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71\") failed: Property not found. retry in 828.6031ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:03.917343466Z","hostname":"buskin","pid":1661,"zone":"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78\") failed: Property not found. retry in 873.115463ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:03.940457525Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a\") failed: Property not found. retry in 1.21081196s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:03.949354083Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1\") failed: Property not found. retry in 1.444483344s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:04.032178765Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b\") failed: Property not found. retry in 922.260049ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:04.043210038Z","hostname":"buskin","pid":1661,"zone":"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827\") failed: Property not found. retry in 848.735646ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:04.504117439Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e\") failed: Property not found. retry in 924.640344ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:04.51509138Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18\") failed: Property not found. retry in 1.041817812s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:04.560308285Z","hostname":"buskin","pid":1661,"zone":"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\") failed: Property not found. retry in 680.391846ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:04.569484613Z","hostname":"buskin","pid":1661,"zone":"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\") failed: Property not found. retry in 808.034715ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:04.731474795Z","hostname":"buskin","pid":1661,"zone":"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71\") failed: Property not found. retry in 1.319056463s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:04.755962025Z","hostname":"buskin","pid":1661,"zone":"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c\") failed: Property not found. retry in 1.07408154s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:04.793948332Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78\") failed: Property not found. retry in 1.10297055s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:04.822918037Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b\") failed: Property not found. retry in 966.095257ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:04.837338153Z","hostname":"buskin","pid":1661,"zone":"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b\") failed: Property not found. retry in 572.980363ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:04.976034602Z","hostname":"buskin","pid":1661,"zone":"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4\") failed: Property not found. retry in 604.128855ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:05.030443405Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a\") failed: Property not found. retry in 704.020989ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:05.171103844Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a\") failed: Property not found. retry in 1.43702013s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:05.232668603Z","hostname":"buskin","pid":1661,"zone":"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\") failed: Property not found. retry in 1.224452147s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:05.261580436Z","hostname":"buskin","pid":1661,"zone":"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e\") failed: Property not found. retry in 1.247790316s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:05.451343108Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1\") failed: Property not found. retry in 1.000794449s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:05.488275443Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\") failed: Property not found. retry in 1.467652605s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:05.549937555Z","hostname":"buskin","pid":1661,"zone":"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b\") failed: Property not found. retry in 1.077985101s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:05.557995148Z","hostname":"buskin","pid":1661,"zone":"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18\") failed: Property not found. retry in 730.409211ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:05.61268255Z","hostname":"buskin","pid":1661,"zone":"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4\") failed: Property not found. retry in 987.837651ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:05.644753949Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b\") failed: Property not found. retry in 1.04087232s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:05.81432281Z","hostname":"buskin","pid":1661,"zone":"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a\") failed: Property not found. retry in 945.572867ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:05.894921814Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78\") failed: Property not found. retry in 1.145972656s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:05.938518331Z","hostname":"buskin","pid":1661,"zone":"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:07.134603236Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2908","zone":"oxz_nexus_0a835148-a2f0-40d4-bc66-05f1d779465b"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:07.134665869Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2908","zone":"oxz_nexus_2bea071a-6ee2-45ba-9a69-93a0e2e5b31b"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:07.134676599Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2908","zone":"oxz_crucible_pantry_e79fbd89-0714-4c5d-b550-de2def94225f"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:07.134682609Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2908","zone":"oxz_external_dns_472c8e8c-ac8f-4193-921c-49dd3ceb288b"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:07.134688329Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2908","zone":"oxz_external_dns_e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:07.134693979Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2908","zone":"oxz_clickhouse_ca2e0ff9-5aa1-4fc2-8f1c-d28eb849db5a"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:07.134699639Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2908","zone":"oxz_crucible_pantry_afeebd42-9418-4a17-adf1-4ce6f3cadd20"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:07.134705Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2908","zone":"oxz_crucible_56d5a9d2-63d5-49e7-a77d-85aa3c6cc7d4"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:07.13471337Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2908","zone":"oxz_crucible_9a6f2c90-9df5-4d09-be49-6a0e3387814a"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:07.13473414Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2908","zone":"oxz_nexus_761d7d83-4f44-4dad-ba8f-694aeae46e71"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:07.134739751Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2908","zone":"oxz_crucible_36c43d98-b274-450d-98d0-db7e0ee7514c"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:07.134745021Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2908","zone":"oxz_oximeter_8dc5039d-cb88-49dd-8ab8-2ca47e481b18"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:07.134750541Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2908","zone":"oxz_crucible_pantry_16f9d651-64fc-4e3c-a69b-bda36e263e78"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:07.134759291Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2908","zone":"oxz_crucible_9227e547-c76f-4bc5-b6f2-101d008d915e"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:07.134766561Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2908","zone":"oxz_crucible_0c085ad2-e5e8-4683-9fed-dd84bc890e5b"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:07.134774692Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2908","zone":"oxz_crucible_fca244c4-2682-42f7-9297-53c69ba4caa1"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:07.134781362Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2908","zone":"oxz_crucible_5eefa476-ecad-43d0-a297-e34591893cf5"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:07.147453405Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2908","zone":"oxz_crucible_a18cc15b-758a-467a-8a3b-ed115f6e1135"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:07.147469535Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"sled-agent/src/services.rs:2908","zone":"oxz_crucible_608a3190-2550-47a2-a23f-81797b571827"} {"msg":"Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:07.148024612Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:198"} {"msg":"Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:07.169881602Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:198"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:07.201578939Z","hostname":"buskin","pid":1661,"uri":"/omicron-zones","method":"PUT","req_id":"58c5d081-b3c5-4e94-89fc-9c1c91fc8ae5","remote_addr":"[fd00:1122:3344:101::1]:58669","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":35097537,"response_code":"204"} {"msg":"Finished setting up services","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:07.319216563Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1059"} {"msg":"Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:07.319889483Z","hostname":"buskin","pid":1661,"component":"RSS","file":"common/src/ledger.rs:198"} {"msg":"Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:07.320114499Z","hostname":"buskin","pid":1661,"component":"RSS","file":"common/src/ledger.rs:198"} {"msg":"Handing off control to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:07.335152124Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:549"} {"msg":"Nexus address: [fd00:1122:3344:101::b]:12221","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:07.335171894Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:551"} {"msg":"rack_network_config: RackNetworkConfigV1 {\n bgp: [],\n infra_ip_first: 192.168.1.30,\n infra_ip_last: 192.168.1.30,\n ports: [\n PortConfigV1 {\n addresses: [\n V4(\n Ipv4Network {\n addr: 10.151.1.174,\n prefix: 24,\n },\n ),\n ],\n autoneg: false,\n bgp_peers: [],\n port: \"qsfp0\",\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Network {\n addr: 0.0.0.0,\n prefix: 0,\n },\n ),\n nexthop: 10.151.1.100,\n },\n ],\n switch: Switch0,\n uplink_port_fec: None,\n uplink_port_speed: Speed40G,\n },\n ],\n rack_subnet: Ipv6Network {\n addr: fd00:1122:3344:100::,\n prefix: 56,\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:07.428656678Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/90130517-c703-4437-bd46-ac71a8336624/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:09.874324553Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/90130517-c703-4437-bd46-ac71a8336624/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:09.934173651Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/90130517-c703-4437-bd46-ac71a8336624/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:10.007383622Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/90130517-c703-4437-bd46-ac71a8336624/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:10.105302069Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/90130517-c703-4437-bd46-ac71a8336624/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:10.214509588Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/90130517-c703-4437-bd46-ac71a8336624/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:10.325985825Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/90130517-c703-4437-bd46-ac71a8336624/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:10.692061571Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/90130517-c703-4437-bd46-ac71a8336624/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:11.122357257Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/90130517-c703-4437-bd46-ac71a8336624/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:11.454109747Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/90130517-c703-4437-bd46-ac71a8336624/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:11.762749569Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/90130517-c703-4437-bd46-ac71a8336624/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:12.494746607Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/90130517-c703-4437-bd46-ac71a8336624/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:13.250550944Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:13.948775143Z","hostname":"buskin","pid":1661,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:45242"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:13.959139266Z","hostname":"buskin","pid":1661,"uri":"/baseboard","method":"GET","req_id":"20a5d81b-2ec0-4b30-9164-893e6d9c2c9d","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:45242","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":191,"response_code":"200"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/90130517-c703-4437-bd46-ac71a8336624/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:14.29336727Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/90130517-c703-4437-bd46-ac71a8336624/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:15.254891621Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/90130517-c703-4437-bd46-ac71a8336624/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:16.743702847Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/90130517-c703-4437-bd46-ac71a8336624/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:17.691573245Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"failed to notify nexus about a new pool oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:17.817246591Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"215.325962545s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104/zpools/14b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: cac4b398-c394-4c89-a5de-5577a51ce104, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:17.818005374Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"215.327523191s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:17.818284242Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"215.325851561s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104/zpools/d462a7f7-b628-40fe-80ff-4e4189e2d62b): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: cac4b398-c394-4c89-a5de-5577a51ce104, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:17.818452417Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"215.514722266s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:17.818639803Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"215.326455819s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104/zpools/cd70d7f6-2354-4bf2-8012-55bf9eaf7930): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:17.819427567Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:17.825002195Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:17.827980715Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:17.831082429Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:17.834747909Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:17.837727639Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:17.840638187Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:17.843143533Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:17.846242387Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:17.84834548Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:17.85100674Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:17.853137135Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:17.855862137Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:17.858123075Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:17.860060904Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:17.862607911Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:17.864830448Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:17.867273182Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:17.86987353Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:17.872013535Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:17.874989935Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:17.877421968Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:17.879650836Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:17.883611575Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:17.88607799Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:17.889318708Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:17.891671169Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:17.894108772Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:17.896707501Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:17.899237557Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:17.901559737Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:17.904364382Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:17.907099745Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:17.909523458Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:17.911961852Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:19.043640631Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:19.046652852Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:19.050456807Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:19.052998724Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:19.055271523Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:19.058336965Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:19.060853531Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:19.063330076Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:19.065754579Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:19.068934705Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:19.071770381Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:19.074218335Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:19.077167604Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:19.079705441Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:19.083032901Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:19.08530474Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:19.088083414Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:19.090594589Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:19.093415615Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:19.097194529Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:19.100610962Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: cac4b398-c394-4c89-a5de-5577a51ce104, variant: M2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:19.103980904Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"216.799581011s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:19.103998524Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:19.106996115Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:19.109533051Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:19.112294025Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:19.114727938Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:19.117148211Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:19.119311707Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:19.122277976Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:19.124999118Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:19.1290097Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:19.131386022Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/90130517-c703-4437-bd46-ac71a8336624/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:19.82365093Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:20.263651859Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:20.266418903Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:20.269959279Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:20.273106205Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:20.27593371Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:20.278314952Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:20.281008733Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:20.283310933Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:20.286459078Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:20.289160049Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:20.292340095Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:20.295079428Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:20.298365397Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:20.301530113Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:20.304127331Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:20.306665398Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:20.309352789Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:20.311965888Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:20.315056641Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:20.317485675Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:20.321276889Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:20.32394746Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:20.326061154Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:20.328426145Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:20.330955852Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:20.334009214Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:20.336741896Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:20.339951953Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:20.342530641Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:20.345731048Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:20.348981106Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:20.351605756Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:20.354308597Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:20.3573909Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:20.359992169Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:20.363438133Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:20.366091213Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:20.368453634Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:20.371364762Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cac4b398-c394-4c89-a5de-5577a51ce104): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:20.373636071Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:616"} {"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:21.016577739Z","hostname":"buskin","pid":1661,"sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1120","retry_after":"8.149744651s","error":"\"Metric registration error: Error Response: status: 503 Service Unavailable; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"ec06667f-87ff-4e15-a219-d173014135ef\\\", \\\"content-length\\\": \\\"133\\\", \\\"date\\\": \\\"Thu, 07 Mar 2024 22:47:20 GMT\\\"}; value: Error { error_code: Some(\\\"ServiceNotAvailable\\\"), message: \\\"Service Unavailable\\\", request_id: \\\"ec06667f-87ff-4e15-a219-d173014135ef\\\" }\""} {"msg":"failed to notify nexus about a new pool oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:21.022019353Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"218.530710824s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"41bf8003-72e1-454c-bb23-cd4c1b8aba73\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Thu, 07 Mar 2024 22:47:21 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"41bf8003-72e1-454c-bb23-cd4c1b8aba73\\\" }\""} {"msg":"failed to notify nexus about a new pool oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:21.024607491Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"218.532110737s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"78dcc579-c4f1-46e6-a65a-30f22a021ba4\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Thu, 07 Mar 2024 22:47:20 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"78dcc579-c4f1-46e6-a65a-30f22a021ba4\\\" }\""} {"msg":"failed to notify nexus about a new pool oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:21.026693925Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"218.534447417s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"dff92480-448f-462d-9a1a-e4278c586fcb\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Thu, 07 Mar 2024 22:47:21 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"dff92480-448f-462d-9a1a-e4278c586fcb\\\" }\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", sled_id: cac4b398-c394-4c89-a5de-5577a51ce104, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:21.050253586Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"218.745984586s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"70e1fef9-d8e3-484b-be26-d9285e3d58bf\\\", \\\"content-length\\\": \\\"173\\\", \\\"date\\\": \\\"Thu, 07 Mar 2024 22:47:21 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: sled with id \\\\\\\"cac4b398-c394-4c89-a5de-5577a51ce104\\\\\\\"\\\", request_id: \\\"70e1fef9-d8e3-484b-be26-d9285e3d58bf\\\" }\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", sled_id: cac4b398-c394-4c89-a5de-5577a51ce104, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:21.062495996Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"218.757884225s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"49edc2bf-54c4-4ebc-bd69-6013b7b35dbf\\\", \\\"content-length\\\": \\\"173\\\", \\\"date\\\": \\\"Thu, 07 Mar 2024 22:47:21 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: sled with id \\\\\\\"cac4b398-c394-4c89-a5de-5577a51ce104\\\\\\\"\\\", request_id: \\\"49edc2bf-54c4-4ebc-bd69-6013b7b35dbf\\\" }\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", sled_id: cac4b398-c394-4c89-a5de-5577a51ce104, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:21.069623271Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"218.764956929s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"9c95d674-800b-456f-8bc6-8f7f4cc30815\\\", \\\"content-length\\\": \\\"173\\\", \\\"date\\\": \\\"Thu, 07 Mar 2024 22:47:20 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: sled with id \\\\\\\"cac4b398-c394-4c89-a5de-5577a51ce104\\\\\\\"\\\", request_id: \\\"9c95d674-800b-456f-8bc6-8f7f4cc30815\\\" }\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: cac4b398-c394-4c89-a5de-5577a51ce104, variant: M2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:21.080285463Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"218.775840888s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"edd07ea3-3118-4ea1-815b-1eb646a2d035\\\", \\\"content-length\\\": \\\"173\\\", \\\"date\\\": \\\"Thu, 07 Mar 2024 22:47:21 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: sled with id \\\\\\\"cac4b398-c394-4c89-a5de-5577a51ce104\\\\\\\"\\\", request_id: \\\"edd07ea3-3118-4ea1-815b-1eb646a2d035\\\" }\""} {"msg":"failed to notify nexus about a new pool oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:21.082148569Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"218.590509201s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"cddbe663-0edf-4f6b-bda6-d2e937746c60\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Thu, 07 Mar 2024 22:47:20 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"cddbe663-0edf-4f6b-bda6-d2e937746c60\\\" }\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: cac4b398-c394-4c89-a5de-5577a51ce104, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:21.088723998Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"218.784745237s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"810cf180-392d-4413-8538-cde38ab3c40a\\\", \\\"content-length\\\": \\\"173\\\", \\\"date\\\": \\\"Thu, 07 Mar 2024 22:47:21 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: sled with id \\\\\\\"cac4b398-c394-4c89-a5de-5577a51ce104\\\\\\\"\\\", request_id: \\\"810cf180-392d-4413-8538-cde38ab3c40a\\\" }\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", sled_id: cac4b398-c394-4c89-a5de-5577a51ce104, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:21.095466892Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"218.791331896s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"4497af51-d212-44f9-8c41-23fd4e15beea\\\", \\\"content-length\\\": \\\"173\\\", \\\"date\\\": \\\"Thu, 07 Mar 2024 22:47:20 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: sled with id \\\\\\\"cac4b398-c394-4c89-a5de-5577a51ce104\\\\\\\"\\\", request_id: \\\"4497af51-d212-44f9-8c41-23fd4e15beea\\\" }\""} {"msg":"failed to notify nexus about a new pool oxp_31bd71cd-4736-4a12-a387-9b74b050396f on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:21.097939597Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"218.606499684s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"b494b7a5-6429-47e7-8f2c-3b19a53c8579\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Thu, 07 Mar 2024 22:47:20 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"b494b7a5-6429-47e7-8f2c-3b19a53c8579\\\" }\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: cac4b398-c394-4c89-a5de-5577a51ce104, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:21.105715751Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"218.801031708s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"2f0d799f-4511-4ba4-81bd-3873f1cd6853\\\", \\\"content-length\\\": \\\"173\\\", \\\"date\\\": \\\"Thu, 07 Mar 2024 22:47:21 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: sled with id \\\\\\\"cac4b398-c394-4c89-a5de-5577a51ce104\\\\\\\"\\\", request_id: \\\"2f0d799f-4511-4ba4-81bd-3873f1cd6853\\\" }\""} {"msg":"failed to notify nexus about a new pool oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:21.108305999Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"218.616524646s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"bb959f1b-9514-464b-ac33-5c3f07910c2b\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Thu, 07 Mar 2024 22:47:21 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"bb959f1b-9514-464b-ac33-5c3f07910c2b\\\" }\""} {"msg":"failed to notify nexus about a new pool oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:21.11063905Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"218.618681641s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"9581d35c-09ea-4f08-92cc-9969b621c74e\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Thu, 07 Mar 2024 22:47:21 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"9581d35c-09ea-4f08-92cc-9969b621c74e\\\" }\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: cac4b398-c394-4c89-a5de-5577a51ce104, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:21.123478628Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"218.632641863s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"7330f1cd-5983-4c3b-8270-ab22d19285de\\\", \\\"content-length\\\": \\\"173\\\", \\\"date\\\": \\\"Thu, 07 Mar 2024 22:47:20 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: sled with id \\\\\\\"cac4b398-c394-4c89-a5de-5577a51ce104\\\\\\\"\\\", request_id: \\\"7330f1cd-5983-4c3b-8270-ab22d19285de\\\" }\""} {"msg":"failed to notify nexus about a new pool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:21.125520319Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"218.632916722s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"10d1935d-45e5-46e1-8a86-0d3ee45505d6\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Thu, 07 Mar 2024 22:47:21 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"10d1935d-45e5-46e1-8a86-0d3ee45505d6\\\" }\""} {"msg":"failed to notify nexus about a new pool oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:21.127028695Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"218.634912591s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"0a5007d5-eee7-4d29-8ab5-1245b3e6c462\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Thu, 07 Mar 2024 22:47:20 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"0a5007d5-eee7-4d29-8ab5-1245b3e6c462\\\" }\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: cac4b398-c394-4c89-a5de-5577a51ce104, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:21.141918385Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"218.651498652s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"5925e13d-b1a3-4ec7-9cb8-77c3c0526009\\\", \\\"content-length\\\": \\\"173\\\", \\\"date\\\": \\\"Thu, 07 Mar 2024 22:47:20 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: sled with id \\\\\\\"cac4b398-c394-4c89-a5de-5577a51ce104\\\\\\\"\\\", request_id: \\\"5925e13d-b1a3-4ec7-9cb8-77c3c0526009\\\" }\""} {"msg":"failed to notify nexus about a new pool oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:21.143366289Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"218.651002948s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"73077835-d73a-474e-8d69-17565a51989d\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Thu, 07 Mar 2024 22:47:21 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"73077835-d73a-474e-8d69-17565a51989d\\\" }\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: cac4b398-c394-4c89-a5de-5577a51ce104, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:21.149587756Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"218.845925325s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"d14c1456-3663-4def-804d-e7b43ea04e05\\\", \\\"content-length\\\": \\\"173\\\", \\\"date\\\": \\\"Thu, 07 Mar 2024 22:47:21 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: sled with id \\\\\\\"cac4b398-c394-4c89-a5de-5577a51ce104\\\\\\\"\\\", request_id: \\\"d14c1456-3663-4def-804d-e7b43ea04e05\\\" }\""} {"msg":"failed to notify nexus about a new pool oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:21.151618968Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"218.660516905s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"0e837176-c315-4550-b8b4-3a3e0b595a33\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Thu, 07 Mar 2024 22:47:21 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"0e837176-c315-4550-b8b4-3a3e0b595a33\\\" }\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: cac4b398-c394-4c89-a5de-5577a51ce104, variant: M2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:21.159853336Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"218.855490524s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"9c2da656-5f55-4cda-9c4f-8f44db364d18\\\", \\\"content-length\\\": \\\"173\\\", \\\"date\\\": \\\"Thu, 07 Mar 2024 22:47:21 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: sled with id \\\\\\\"cac4b398-c394-4c89-a5de-5577a51ce104\\\\\\\"\\\", request_id: \\\"9c2da656-5f55-4cda-9c4f-8f44db364d18\\\" }\""} {"msg":"Successfully put SledAgentInfo to nexus","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:21.238688797Z","hostname":"buskin","pid":1661,"component":"NexusNotifierTask","sled_id":"cac4b398-c394-4c89-a5de-5577a51ce104","component":"SledAgent","file":"sled-agent/src/nexus.rs:589","new_generation":"1","old_generation":"1"} {"msg":"failed to notify nexus about a new pool oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:21.399493614Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"218.90699989s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"070a83aa-3b62-429a-a1a3-6b5e6fa507d9\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Thu, 07 Mar 2024 22:47:21 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"070a83aa-3b62-429a-a1a3-6b5e6fa507d9\\\" }\""} {"msg":"failed to notify nexus about a new pool oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:21.506768654Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"219.014524566s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"fbcce7f0-eaf3-44ca-bfd0-fc57a0f2e0b8\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Thu, 07 Mar 2024 22:47:21 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"fbcce7f0-eaf3-44ca-bfd0-fc57a0f2e0b8\\\" }\""} {"msg":"failed to notify nexus about a new pool oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:21.539260295Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"219.048157242s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"46adeb82-f069-4623-977c-1291e4208786\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Thu, 07 Mar 2024 22:47:21 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"46adeb82-f069-4623-977c-1291e4208786\\\" }\""} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:21.594521464Z","hostname":"buskin","pid":1661,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:64728"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:21.59539041Z","hostname":"buskin","pid":1661,"uri":"/inventory","method":"GET","req_id":"9820243f-6e69-42f1-b46a-1b0f7fb54e9d","remote_addr":"[fd00:1122:3344:101::c]:64728","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":697,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:21.596404341Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:21.596749531Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:21.600471244Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:21.600842845Z","hostname":"buskin","pid":1661,"uri":"/omicron-zones","method":"GET","req_id":"6d2940f0-6607-4d78-a368-9a76f4333bb6","remote_addr":"[fd00:1122:3344:101::c]:64728","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":5118,"response_code":"200"} {"msg":"failed to notify nexus about a new pool oxp_31bd71cd-4736-4a12-a387-9b74b050396f on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:21.612460656Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"219.121030333s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"590cc33b-3741-4dbf-b320-d2ce323bc25f\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Thu, 07 Mar 2024 22:47:21 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"590cc33b-3741-4dbf-b320-d2ce323bc25f\\\" }\""} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:21.623128598Z","hostname":"buskin","pid":1661,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:34258"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:21.626574122Z","hostname":"buskin","pid":1661,"uri":"/network-bootstore-config","method":"GET","req_id":"738a316c-7b9c-4932-bc41-3227c00cc3b4","remote_addr":"[fd00:1122:3344:101::c]:34258","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":3377,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:21.626615924Z","hostname":"buskin","pid":1661,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:43450"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:21.626623874Z","hostname":"buskin","pid":1661,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:44758"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:21.626663015Z","hostname":"buskin","pid":1661,"uri":"/inventory","method":"GET","req_id":"393c7064-6330-49b9-a4dd-e9d9d3d2973f","remote_addr":"[fd00:1122:3344:101::b]:43450","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":25,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:21.626872781Z","hostname":"buskin","pid":1661,"uri":"/network-bootstore-config","method":"GET","req_id":"201efb47-a63f-4048-bc76-baa6f2db86b5","remote_addr":"[fd00:1122:3344:101::b]:44758","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":66,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:21.62748445Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:21.627513401Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:21.627727817Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:21.627954054Z","hostname":"buskin","pid":1661,"uri":"/omicron-zones","method":"GET","req_id":"ea1741ab-ad9a-4f13-a906-009c2b0fe2eb","remote_addr":"[fd00:1122:3344:101::b]:43450","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1055,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:21.653704362Z","hostname":"buskin","pid":1661,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:53319"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:21.653771344Z","hostname":"buskin","pid":1661,"uri":"/network-bootstore-config","method":"GET","req_id":"37b26bbc-f353-44c8-baef-26f31486d2ac","remote_addr":"[fd00:1122:3344:101::a]:53319","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":47,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:21.653940479Z","hostname":"buskin","pid":1661,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:34953"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:21.654183446Z","hostname":"buskin","pid":1661,"uri":"/inventory","method":"GET","req_id":"1ba98966-c82e-47aa-9488-c1ce202a1717","remote_addr":"[fd00:1122:3344:101::a]:34953","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":34,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:21.654383962Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:21.654397713Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:21.654558237Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:21.656222188Z","hostname":"buskin","pid":1661,"uri":"/omicron-zones","method":"GET","req_id":"1586dc11-dd66-4a97-ab9a-0d9cee23ded3","remote_addr":"[fd00:1122:3344:101::a]:34953","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1853,"response_code":"200"} {"msg":"failed to notify nexus about a new pool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:21.720698595Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"219.228108718s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"7535e763-ef64-4b40-98b0-41321f4cbc35\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Thu, 07 Mar 2024 22:47:21 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"7535e763-ef64-4b40-98b0-41321f4cbc35\\\" }\""} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: cac4b398-c394-4c89-a5de-5577a51ce104, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:21.846543966Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"failed to notify nexus about a new pool oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:21.852339071Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"219.360236007s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"72db2cd9-5aa7-433c-94c8-c4c29fbbb2ee\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Thu, 07 Mar 2024 22:47:21 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"72db2cd9-5aa7-433c-94c8-c4c29fbbb2ee\\\" }\""} {"msg":"failed to notify nexus about a new pool oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:21.922531211Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"219.430770108s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"168279af-7c5e-4d51-a29a-a3688e44ac48\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Thu, 07 Mar 2024 22:47:21 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"168279af-7c5e-4d51-a29a-a3688e44ac48\\\" }\""} {"msg":"failed to notify nexus about a new pool oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:47:21.927256644Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"219.435968324s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"7c4566d0-01e2-440e-9a9d-3cf28ee91ad1\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Thu, 07 Mar 2024 22:47:21 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"7c4566d0-01e2-440e-9a9d-3cf28ee91ad1\\\" }\""} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: cac4b398-c394-4c89-a5de-5577a51ce104, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:22.006169337Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", sled_id: cac4b398-c394-4c89-a5de-5577a51ce104, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:22.006182708Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", sled_id: cac4b398-c394-4c89-a5de-5577a51ce104, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:22.006196058Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: cac4b398-c394-4c89-a5de-5577a51ce104, variant: M2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:22.252481687Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: cac4b398-c394-4c89-a5de-5577a51ce104, variant: M2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:22.252501937Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", sled_id: cac4b398-c394-4c89-a5de-5577a51ce104, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:22.252506928Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", sled_id: cac4b398-c394-4c89-a5de-5577a51ce104, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:22.252510988Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: cac4b398-c394-4c89-a5de-5577a51ce104, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:22.252515248Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: cac4b398-c394-4c89-a5de-5577a51ce104, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:22.252519578Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: cac4b398-c394-4c89-a5de-5577a51ce104, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:22.252523578Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:22.441464025Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"78e37f1c-f75d-41db-b866-45121c66cf54\", \"content-length\": \"174\", \"date\": \"Thu, 07 Mar 2024 22:47:22 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: zpool with id \\\"24b4dc87-ab46-49fb-a4b4-d361ae214c03\\\"\", request_id: \"78e37f1c-f75d-41db-b866-45121c66cf54\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:22.951717455Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:23.055283633Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:23.055303264Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:23.055308584Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:23.055312964Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:23.055317104Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:23.055321144Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:23.055325965Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:23.055331085Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:23.055334995Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:23.055345255Z","hostname":"buskin","pid":1661,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:27.020625158Z","hostname":"buskin","pid":1661,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:62892"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:27.02666082Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0b3b3eae-a95f-4091-984a-93fccf167e79, kind: Service { id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c }, name: Name(\"external-dns-e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: ca9c5211-aa62-4394-a496-e20bf51739bc, kind: Service { id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b }, name: Name(\"external-dns-472c8e8c-ac8f-4193-921c-49dd3ceb288b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1eeeae19-992a-47f8-9e7b-dc93c05f0958, kind: Service { id: 761d7d83-4f44-4dad-ba8f-694aeae46e71 }, name: Name(\"nexus-761d7d83-4f44-4dad-ba8f-694aeae46e71\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 89eadaf9-099e-4929-ad88-e7ee2258e13d, kind: Service { id: 0a835148-a2f0-40d4-bc66-05f1d779465b }, name: Name(\"nexus-0a835148-a2f0-40d4-bc66-05f1d779465b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 72])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 8c296dc4-9b38-40c8-9549-80b2967f6cdc, kind: Service { id: 2bea071a-6ee2-45ba-9a69-93a0e2e5b31b }, name: Name(\"nexus-2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:27.027513506Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:27.027685271Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:27.027702911Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:27.027707281Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:27.027711701Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:27.027716002Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:27.027743662Z","hostname":"buskin","pid":1661,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"6491a799-ca25-4e98-af7a-f9dd939660c1","remote_addr":"[fd00:1122:3344:101::b]:62892","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":6755,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2fe0467c-d615-454e-9c54-a4b57e07b2f5\", \"content-length\": \"127\", \"date\": \"Thu, 07 Mar 2024 22:47:27 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"2fe0467c-d615-454e-9c54-a4b57e07b2f5\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:27.20810118Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:27.390034625Z","hostname":"buskin","pid":1661,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:47460"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:27.390088357Z","hostname":"buskin","pid":1661,"uri":"/inventory","method":"GET","req_id":"1b29c8c0-49b8-4b73-b9db-531e3ac2b3a4","remote_addr":"[fd00:1122:3344:101::b]:47460","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":33,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:27.390263342Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:27.390282002Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:27.390489479Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:27.390675244Z","hostname":"buskin","pid":1661,"uri":"/omicron-zones","method":"GET","req_id":"04cbf8e6-8451-4880-b1a3-ebac09507f1b","remote_addr":"[fd00:1122:3344:101::b]:47460","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":435,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:29.203866287Z","hostname":"buskin","pid":1661,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:63999"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:29.205201138Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0b3b3eae-a95f-4091-984a-93fccf167e79, kind: Service { id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c }, name: Name(\"external-dns-e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: ca9c5211-aa62-4394-a496-e20bf51739bc, kind: Service { id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b }, name: Name(\"external-dns-472c8e8c-ac8f-4193-921c-49dd3ceb288b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1eeeae19-992a-47f8-9e7b-dc93c05f0958, kind: Service { id: 761d7d83-4f44-4dad-ba8f-694aeae46e71 }, name: Name(\"nexus-761d7d83-4f44-4dad-ba8f-694aeae46e71\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 89eadaf9-099e-4929-ad88-e7ee2258e13d, kind: Service { id: 0a835148-a2f0-40d4-bc66-05f1d779465b }, name: Name(\"nexus-0a835148-a2f0-40d4-bc66-05f1d779465b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 72])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 8c296dc4-9b38-40c8-9549-80b2967f6cdc, kind: Service { id: 2bea071a-6ee2-45ba-9a69-93a0e2e5b31b }, name: Name(\"nexus-2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:29.205233439Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:29.2052634Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:29.20527019Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:29.2052766Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:29.20528633Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:29.20529759Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:29.205324431Z","hostname":"buskin","pid":1661,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"3665249d-5256-48ed-b136-5fe59c87f05a","remote_addr":"[fd00:1122:3344:101::b]:63999","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1395,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2145d462-84dc-440b-bd9f-79b8c540b562\", \"content-length\": \"127\", \"date\": \"Thu, 07 Mar 2024 22:47:29 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"2145d462-84dc-440b-bd9f-79b8c540b562\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:29.288527544Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:29.434567875Z","hostname":"buskin","pid":1661,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:47701"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:29.434634257Z","hostname":"buskin","pid":1661,"uri":"/inventory","method":"GET","req_id":"8722046b-0c7f-44cb-912b-3497ff939199","remote_addr":"[fd00:1122:3344:101::b]:47701","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":41,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:29.43538382Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:29.435402751Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:29.435601126Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:29.435795132Z","hostname":"buskin","pid":1661,"uri":"/omicron-zones","method":"GET","req_id":"5fba3965-2264-4e1f-84de-4903127284fb","remote_addr":"[fd00:1122:3344:101::b]:47701","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1037,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:33.1754495Z","hostname":"buskin","pid":1661,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:56007"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:33.175722308Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0b3b3eae-a95f-4091-984a-93fccf167e79, kind: Service { id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c }, name: Name(\"external-dns-e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: ca9c5211-aa62-4394-a496-e20bf51739bc, kind: Service { id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b }, name: Name(\"external-dns-472c8e8c-ac8f-4193-921c-49dd3ceb288b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1eeeae19-992a-47f8-9e7b-dc93c05f0958, kind: Service { id: 761d7d83-4f44-4dad-ba8f-694aeae46e71 }, name: Name(\"nexus-761d7d83-4f44-4dad-ba8f-694aeae46e71\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 89eadaf9-099e-4929-ad88-e7ee2258e13d, kind: Service { id: 0a835148-a2f0-40d4-bc66-05f1d779465b }, name: Name(\"nexus-0a835148-a2f0-40d4-bc66-05f1d779465b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 72])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 8c296dc4-9b38-40c8-9549-80b2967f6cdc, kind: Service { id: 2bea071a-6ee2-45ba-9a69-93a0e2e5b31b }, name: Name(\"nexus-2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:33.175766039Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:33.17579858Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:33.17580841Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:33.175816891Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:33.175826511Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:33.175836201Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:33.175875023Z","hostname":"buskin","pid":1661,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"7dd4a71b-75fd-49bf-bd2a-a03d2510f2c8","remote_addr":"[fd00:1122:3344:101::b]:56007","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":323,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0210eacb-f7bf-4ca8-90e9-ff3b6eaf9666\", \"content-length\": \"127\", \"date\": \"Thu, 07 Mar 2024 22:47:33 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"0210eacb-f7bf-4ca8-90e9-ff3b6eaf9666\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:33.234321048Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:33.407073115Z","hostname":"buskin","pid":1661,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:63337"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:33.407155008Z","hostname":"buskin","pid":1661,"uri":"/inventory","method":"GET","req_id":"0772510c-861d-4519-bf6b-f7a2af074f08","remote_addr":"[fd00:1122:3344:101::b]:63337","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":38,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:33.40789102Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:33.407910241Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:33.408100686Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:33.408288162Z","hostname":"buskin","pid":1661,"uri":"/omicron-zones","method":"GET","req_id":"44f0086c-aa5e-4f51-bacf-7a26dce7b793","remote_addr":"[fd00:1122:3344:101::b]:63337","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":914,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:37.648914019Z","hostname":"buskin","pid":1661,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:37404"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:37.64960151Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0b3b3eae-a95f-4091-984a-93fccf167e79, kind: Service { id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c }, name: Name(\"external-dns-e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: ca9c5211-aa62-4394-a496-e20bf51739bc, kind: Service { id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b }, name: Name(\"external-dns-472c8e8c-ac8f-4193-921c-49dd3ceb288b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1eeeae19-992a-47f8-9e7b-dc93c05f0958, kind: Service { id: 761d7d83-4f44-4dad-ba8f-694aeae46e71 }, name: Name(\"nexus-761d7d83-4f44-4dad-ba8f-694aeae46e71\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 89eadaf9-099e-4929-ad88-e7ee2258e13d, kind: Service { id: 0a835148-a2f0-40d4-bc66-05f1d779465b }, name: Name(\"nexus-0a835148-a2f0-40d4-bc66-05f1d779465b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 72])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 8c296dc4-9b38-40c8-9549-80b2967f6cdc, kind: Service { id: 2bea071a-6ee2-45ba-9a69-93a0e2e5b31b }, name: Name(\"nexus-2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:37.649646101Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:37.649676172Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:37.649683383Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:37.649688383Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:37.649694773Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:37.649699963Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:37.649713783Z","hostname":"buskin","pid":1661,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"41240251-30b0-4cf2-98a8-63f993e17356","remote_addr":"[fd00:1122:3344:101::b]:37404","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":761,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0c244fad-b9d4-4e71-9d23-40e85ede5fb7\", \"content-length\": \"127\", \"date\": \"Thu, 07 Mar 2024 22:47:37 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"0c244fad-b9d4-4e71-9d23-40e85ede5fb7\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:37.708388565Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:37.83864859Z","hostname":"buskin","pid":1661,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:52768"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:37.838689131Z","hostname":"buskin","pid":1661,"uri":"/inventory","method":"GET","req_id":"157e399d-9d2a-47d7-8059-364d9227fe7d","remote_addr":"[fd00:1122:3344:101::b]:52768","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":33,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:37.838863866Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:37.838882917Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:37.839094283Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:37.839438283Z","hostname":"buskin","pid":1661,"uri":"/omicron-zones","method":"GET","req_id":"7e655e38-d029-40de-b44c-777bce17c8a9","remote_addr":"[fd00:1122:3344:101::b]:52768","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":600,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:43.907797739Z","hostname":"buskin","pid":1661,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:54468"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:43.90881116Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0b3b3eae-a95f-4091-984a-93fccf167e79, kind: Service { id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c }, name: Name(\"external-dns-e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: ca9c5211-aa62-4394-a496-e20bf51739bc, kind: Service { id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b }, name: Name(\"external-dns-472c8e8c-ac8f-4193-921c-49dd3ceb288b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1eeeae19-992a-47f8-9e7b-dc93c05f0958, kind: Service { id: 761d7d83-4f44-4dad-ba8f-694aeae46e71 }, name: Name(\"nexus-761d7d83-4f44-4dad-ba8f-694aeae46e71\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 89eadaf9-099e-4929-ad88-e7ee2258e13d, kind: Service { id: 0a835148-a2f0-40d4-bc66-05f1d779465b }, name: Name(\"nexus-0a835148-a2f0-40d4-bc66-05f1d779465b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 72])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 8c296dc4-9b38-40c8-9549-80b2967f6cdc, kind: Service { id: 2bea071a-6ee2-45ba-9a69-93a0e2e5b31b }, name: Name(\"nexus-2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:43.908840761Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:43.908868602Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:43.908874072Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:43.908878182Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:43.908882912Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:43.908888512Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:43.908909793Z","hostname":"buskin","pid":1661,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"f7e79393-efc1-455f-94a6-54b1964e7d96","remote_addr":"[fd00:1122:3344:101::b]:54468","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1078,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"588eb285-6d5d-4cd7-9201-8462f3e3f129\", \"content-length\": \"127\", \"date\": \"Thu, 07 Mar 2024 22:47:43 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"588eb285-6d5d-4cd7-9201-8462f3e3f129\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:43.967768661Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:44.008700997Z","hostname":"buskin","pid":1661,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:63710"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:44.008760849Z","hostname":"buskin","pid":1661,"uri":"/baseboard","method":"GET","req_id":"9ca76b6c-3acc-4b85-a8a4-467475bcc871","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:63710","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":26,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:44.073524625Z","hostname":"buskin","pid":1661,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:42415"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:44.073592987Z","hostname":"buskin","pid":1661,"uri":"/inventory","method":"GET","req_id":"d0c67d4b-5358-468e-8b63-0274a803bf92","remote_addr":"[fd00:1122:3344:101::b]:42415","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":40,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:44.073776692Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:44.073800623Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:44.074078161Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:44.07435789Z","hostname":"buskin","pid":1661,"uri":"/omicron-zones","method":"GET","req_id":"a8a2f183-e8db-48b2-95c7-5f4478374921","remote_addr":"[fd00:1122:3344:101::b]:42415","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":619,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:49.439092065Z","hostname":"buskin","pid":1661,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:32768"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:49.439218769Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0b3b3eae-a95f-4091-984a-93fccf167e79, kind: Service { id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c }, name: Name(\"external-dns-e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: ca9c5211-aa62-4394-a496-e20bf51739bc, kind: Service { id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b }, name: Name(\"external-dns-472c8e8c-ac8f-4193-921c-49dd3ceb288b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1eeeae19-992a-47f8-9e7b-dc93c05f0958, kind: Service { id: 761d7d83-4f44-4dad-ba8f-694aeae46e71 }, name: Name(\"nexus-761d7d83-4f44-4dad-ba8f-694aeae46e71\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 89eadaf9-099e-4929-ad88-e7ee2258e13d, kind: Service { id: 0a835148-a2f0-40d4-bc66-05f1d779465b }, name: Name(\"nexus-0a835148-a2f0-40d4-bc66-05f1d779465b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 72])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 8c296dc4-9b38-40c8-9549-80b2967f6cdc, kind: Service { id: 2bea071a-6ee2-45ba-9a69-93a0e2e5b31b }, name: Name(\"nexus-2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:49.43926634Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:49.439279891Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:49.439290901Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:49.439299501Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:49.439310472Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:49.439319832Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:49.439336092Z","hostname":"buskin","pid":1661,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"be16eeac-dbf5-4bce-bcdd-6faf2ded6a96","remote_addr":"[fd00:1122:3344:101::b]:32768","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":207,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9ad9947b-e387-4089-9efd-62b4bf1230e3\", \"content-length\": \"127\", \"date\": \"Thu, 07 Mar 2024 22:47:49 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"9ad9947b-e387-4089-9efd-62b4bf1230e3\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:49.488715624Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:49.57831543Z","hostname":"buskin","pid":1661,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:39333"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:49.578367541Z","hostname":"buskin","pid":1661,"uri":"/inventory","method":"GET","req_id":"e7db2183-b15b-46ca-b96e-e974899cc335","remote_addr":"[fd00:1122:3344:101::b]:39333","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":34,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:49.578968659Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:49.57898559Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:49.579171865Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:49.57933999Z","hostname":"buskin","pid":1661,"uri":"/omicron-zones","method":"GET","req_id":"1f98d716-826e-45ba-be74-c041b38adfc6","remote_addr":"[fd00:1122:3344:101::b]:39333","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":845,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:54.306203711Z","hostname":"buskin","pid":1661,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:35548"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:54.306537971Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0b3b3eae-a95f-4091-984a-93fccf167e79, kind: Service { id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c }, name: Name(\"external-dns-e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: ca9c5211-aa62-4394-a496-e20bf51739bc, kind: Service { id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b }, name: Name(\"external-dns-472c8e8c-ac8f-4193-921c-49dd3ceb288b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1eeeae19-992a-47f8-9e7b-dc93c05f0958, kind: Service { id: 761d7d83-4f44-4dad-ba8f-694aeae46e71 }, name: Name(\"nexus-761d7d83-4f44-4dad-ba8f-694aeae46e71\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 89eadaf9-099e-4929-ad88-e7ee2258e13d, kind: Service { id: 0a835148-a2f0-40d4-bc66-05f1d779465b }, name: Name(\"nexus-0a835148-a2f0-40d4-bc66-05f1d779465b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 72])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 8c296dc4-9b38-40c8-9549-80b2967f6cdc, kind: Service { id: 2bea071a-6ee2-45ba-9a69-93a0e2e5b31b }, name: Name(\"nexus-2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:54.306575042Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:54.306602723Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:54.306608653Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:54.306612913Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:54.306618153Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:54.306626594Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:54.306647294Z","hostname":"buskin","pid":1661,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"44fa80af-21fe-4463-ad36-82100bd58f02","remote_addr":"[fd00:1122:3344:101::b]:35548","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":407,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"65651b32-f427-451b-ad0e-a50526243743\", \"content-length\": \"127\", \"date\": \"Thu, 07 Mar 2024 22:47:54 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"65651b32-f427-451b-ad0e-a50526243743\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:54.360997696Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:54.469124211Z","hostname":"buskin","pid":1661,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:53525"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:54.469194364Z","hostname":"buskin","pid":1661,"uri":"/inventory","method":"GET","req_id":"24a0fa75-8f96-4f84-81a1-a20ebe596446","remote_addr":"[fd00:1122:3344:101::b]:53525","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":34,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:54.469374869Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:54.469396539Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:54.469620536Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:54.469822312Z","hostname":"buskin","pid":1661,"uri":"/omicron-zones","method":"GET","req_id":"cb4706b8-3abd-42cc-a031-037c1e30c359","remote_addr":"[fd00:1122:3344:101::b]:53525","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":485,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:59.243772945Z","hostname":"buskin","pid":1661,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::d]:39455"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:47:59.251197699Z","hostname":"buskin","pid":1661,"uri":"/metrics/collect/cac4b398-c394-4c89-a5de-5577a51ce104","method":"GET","req_id":"742ff12c-75f7-4aee-9a88-fef90ea092fd","remote_addr":"[fd00:1122:3344:101::d]:39455","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":7345,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:01.669476475Z","hostname":"buskin","pid":1661,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:58333"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:01.670136255Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0b3b3eae-a95f-4091-984a-93fccf167e79, kind: Service { id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c }, name: Name(\"external-dns-e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: ca9c5211-aa62-4394-a496-e20bf51739bc, kind: Service { id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b }, name: Name(\"external-dns-472c8e8c-ac8f-4193-921c-49dd3ceb288b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1eeeae19-992a-47f8-9e7b-dc93c05f0958, kind: Service { id: 761d7d83-4f44-4dad-ba8f-694aeae46e71 }, name: Name(\"nexus-761d7d83-4f44-4dad-ba8f-694aeae46e71\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 89eadaf9-099e-4929-ad88-e7ee2258e13d, kind: Service { id: 0a835148-a2f0-40d4-bc66-05f1d779465b }, name: Name(\"nexus-0a835148-a2f0-40d4-bc66-05f1d779465b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 72])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 8c296dc4-9b38-40c8-9549-80b2967f6cdc, kind: Service { id: 2bea071a-6ee2-45ba-9a69-93a0e2e5b31b }, name: Name(\"nexus-2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:01.670169966Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:01.670191627Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:01.670198697Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:01.670204157Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:01.670210147Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:01.670219517Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:01.670235298Z","hostname":"buskin","pid":1661,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"93f64985-a439-4231-b7eb-30f0089336d0","remote_addr":"[fd00:1122:3344:101::b]:58333","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":694,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0d483c0e-b1bf-4765-8a8a-6d3154398d04\", \"content-length\": \"127\", \"date\": \"Thu, 07 Mar 2024 22:48:01 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"0d483c0e-b1bf-4765-8a8a-6d3154398d04\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:01.723306731Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:01.813106823Z","hostname":"buskin","pid":1661,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:42203"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:01.813174205Z","hostname":"buskin","pid":1661,"uri":"/inventory","method":"GET","req_id":"d2b8459e-43fa-45ef-98ca-78d4ed928826","remote_addr":"[fd00:1122:3344:101::b]:42203","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":31,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:01.81334892Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:01.813372481Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:01.813566327Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:01.813746792Z","hostname":"buskin","pid":1661,"uri":"/omicron-zones","method":"GET","req_id":"8dea6847-a609-46be-8fdd-9cb510dc610d","remote_addr":"[fd00:1122:3344:101::b]:42203","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":419,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:12.980454738Z","hostname":"buskin","pid":1661,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:42624"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:12.980718456Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0b3b3eae-a95f-4091-984a-93fccf167e79, kind: Service { id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c }, name: Name(\"external-dns-e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: ca9c5211-aa62-4394-a496-e20bf51739bc, kind: Service { id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b }, name: Name(\"external-dns-472c8e8c-ac8f-4193-921c-49dd3ceb288b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1eeeae19-992a-47f8-9e7b-dc93c05f0958, kind: Service { id: 761d7d83-4f44-4dad-ba8f-694aeae46e71 }, name: Name(\"nexus-761d7d83-4f44-4dad-ba8f-694aeae46e71\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 89eadaf9-099e-4929-ad88-e7ee2258e13d, kind: Service { id: 0a835148-a2f0-40d4-bc66-05f1d779465b }, name: Name(\"nexus-0a835148-a2f0-40d4-bc66-05f1d779465b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 72])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 8c296dc4-9b38-40c8-9549-80b2967f6cdc, kind: Service { id: 2bea071a-6ee2-45ba-9a69-93a0e2e5b31b }, name: Name(\"nexus-2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:12.980772527Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:12.980804508Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:12.980815409Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:12.980825989Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:12.98085159Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:12.98086035Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:12.980901161Z","hostname":"buskin","pid":1661,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"e788ef02-148a-4975-bde8-63e1c52bca09","remote_addr":"[fd00:1122:3344:101::b]:42624","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":356,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1f1e4d56-c29c-46a8-86e6-d47b3347f014\", \"content-length\": \"127\", \"date\": \"Thu, 07 Mar 2024 22:48:13 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"1f1e4d56-c29c-46a8-86e6-d47b3347f014\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:13.039024577Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:13.143801611Z","hostname":"buskin","pid":1661,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:52955"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:13.143885324Z","hostname":"buskin","pid":1661,"uri":"/inventory","method":"GET","req_id":"1850ad4b-05be-4386-9737-cc87a8c129ef","remote_addr":"[fd00:1122:3344:101::b]:52955","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":48,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:13.144115731Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:13.144134841Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:13.144381369Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:13.144555824Z","hostname":"buskin","pid":1661,"uri":"/omicron-zones","method":"GET","req_id":"2d86c68a-066d-485b-9771-8421f23fec11","remote_addr":"[fd00:1122:3344:101::b]:52955","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":460,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:14.042120222Z","hostname":"buskin","pid":1661,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:63661"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:14.042164133Z","hostname":"buskin","pid":1661,"uri":"/baseboard","method":"GET","req_id":"668c4d21-4526-431c-ad8d-d7099762bb7e","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:63661","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":32,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:28.701529033Z","hostname":"buskin","pid":1661,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:57137"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:28.701669107Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0b3b3eae-a95f-4091-984a-93fccf167e79, kind: Service { id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c }, name: Name(\"external-dns-e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: ca9c5211-aa62-4394-a496-e20bf51739bc, kind: Service { id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b }, name: Name(\"external-dns-472c8e8c-ac8f-4193-921c-49dd3ceb288b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1eeeae19-992a-47f8-9e7b-dc93c05f0958, kind: Service { id: 761d7d83-4f44-4dad-ba8f-694aeae46e71 }, name: Name(\"nexus-761d7d83-4f44-4dad-ba8f-694aeae46e71\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 89eadaf9-099e-4929-ad88-e7ee2258e13d, kind: Service { id: 0a835148-a2f0-40d4-bc66-05f1d779465b }, name: Name(\"nexus-0a835148-a2f0-40d4-bc66-05f1d779465b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 72])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 8c296dc4-9b38-40c8-9549-80b2967f6cdc, kind: Service { id: 2bea071a-6ee2-45ba-9a69-93a0e2e5b31b }, name: Name(\"nexus-2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:28.701711169Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:28.70174271Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:28.70175099Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:28.70175806Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:28.70176479Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:28.70177123Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:28.701787561Z","hostname":"buskin","pid":1661,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"abf57878-5085-46e6-8e94-5a2e55beb88d","remote_addr":"[fd00:1122:3344:101::b]:57137","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":233,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ada89917-f58e-4e86-8edc-11a24187b23a\", \"content-length\": \"127\", \"date\": \"Thu, 07 Mar 2024 22:48:28 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"ada89917-f58e-4e86-8edc-11a24187b23a\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:28.757339599Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:28.8491068Z","hostname":"buskin","pid":1661,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:61948"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:28.849828952Z","hostname":"buskin","pid":1661,"uri":"/inventory","method":"GET","req_id":"0b9ad5d0-fdb6-4af7-a5e2-8109165ab2c1","remote_addr":"[fd00:1122:3344:101::b]:61948","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":690,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:28.849976296Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:28.850004427Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:28.850262075Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:28.85043751Z","hostname":"buskin","pid":1661,"uri":"/omicron-zones","method":"GET","req_id":"f8f5fed9-757a-4556-8d50-00228f9afbc9","remote_addr":"[fd00:1122:3344:101::b]:61948","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":486,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:29.243536643Z","hostname":"buskin","pid":1661,"uri":"/metrics/collect/cac4b398-c394-4c89-a5de-5577a51ce104","method":"GET","req_id":"55e7e7e5-7919-45a3-a4e3-2dd65200de2c","remote_addr":"[fd00:1122:3344:101::d]:39455","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":289,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:44.074066052Z","hostname":"buskin","pid":1661,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:50041"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:44.07497297Z","hostname":"buskin","pid":1661,"uri":"/baseboard","method":"GET","req_id":"4919867d-6ddc-4e9a-803a-8c1a650c6b10","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:50041","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":677,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:44.305456961Z","hostname":"buskin","pid":1661,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:63606"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:44.305707928Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0b3b3eae-a95f-4091-984a-93fccf167e79, kind: Service { id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c }, name: Name(\"external-dns-e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: ca9c5211-aa62-4394-a496-e20bf51739bc, kind: Service { id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b }, name: Name(\"external-dns-472c8e8c-ac8f-4193-921c-49dd3ceb288b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1eeeae19-992a-47f8-9e7b-dc93c05f0958, kind: Service { id: 761d7d83-4f44-4dad-ba8f-694aeae46e71 }, name: Name(\"nexus-761d7d83-4f44-4dad-ba8f-694aeae46e71\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 89eadaf9-099e-4929-ad88-e7ee2258e13d, kind: Service { id: 0a835148-a2f0-40d4-bc66-05f1d779465b }, name: Name(\"nexus-0a835148-a2f0-40d4-bc66-05f1d779465b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 72])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 8c296dc4-9b38-40c8-9549-80b2967f6cdc, kind: Service { id: 2bea071a-6ee2-45ba-9a69-93a0e2e5b31b }, name: Name(\"nexus-2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:44.30576621Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:44.305981237Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:44.305992387Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:44.306153122Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:44.306167013Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:44.306177173Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:44.306215324Z","hostname":"buskin","pid":1661,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"7d64ef67-27a9-4410-abbc-ae957e2ad2b9","remote_addr":"[fd00:1122:3344:101::b]:63606","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":683,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a0922279-c445-433f-b608-2ff21b6c7d88\", \"content-length\": \"127\", \"date\": \"Thu, 07 Mar 2024 22:48:44 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"a0922279-c445-433f-b608-2ff21b6c7d88\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:44.359183614Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:44.457656368Z","hostname":"buskin","pid":1661,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:56404"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:44.457704809Z","hostname":"buskin","pid":1661,"uri":"/inventory","method":"GET","req_id":"2d3abbbc-aba8-46fe-8c6e-a01e06e5a7c1","remote_addr":"[fd00:1122:3344:101::b]:56404","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":30,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:44.458259056Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:44.458278067Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:44.458656588Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:44.458956717Z","hostname":"buskin","pid":1661,"uri":"/omicron-zones","method":"GET","req_id":"1988256c-86a0-4e18-8d87-295b68985c8f","remote_addr":"[fd00:1122:3344:101::b]:56404","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1134,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:58.933246607Z","hostname":"buskin","pid":1661,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:33334"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:58.933412492Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0b3b3eae-a95f-4091-984a-93fccf167e79, kind: Service { id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c }, name: Name(\"external-dns-e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: ca9c5211-aa62-4394-a496-e20bf51739bc, kind: Service { id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b }, name: Name(\"external-dns-472c8e8c-ac8f-4193-921c-49dd3ceb288b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1eeeae19-992a-47f8-9e7b-dc93c05f0958, kind: Service { id: 761d7d83-4f44-4dad-ba8f-694aeae46e71 }, name: Name(\"nexus-761d7d83-4f44-4dad-ba8f-694aeae46e71\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 89eadaf9-099e-4929-ad88-e7ee2258e13d, kind: Service { id: 0a835148-a2f0-40d4-bc66-05f1d779465b }, name: Name(\"nexus-0a835148-a2f0-40d4-bc66-05f1d779465b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 72])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 8c296dc4-9b38-40c8-9549-80b2967f6cdc, kind: Service { id: 2bea071a-6ee2-45ba-9a69-93a0e2e5b31b }, name: Name(\"nexus-2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:58.933460854Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:58.933477704Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:58.933523526Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:58.933539296Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:58.933547846Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:58.933553516Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:58.933565237Z","hostname":"buskin","pid":1661,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"52907c9d-234b-4769-bd34-8c07b1abfef7","remote_addr":"[fd00:1122:3344:101::b]:33334","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":282,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1ceaa226-10b3-4e27-9531-44b7f32817a5\", \"content-length\": \"127\", \"date\": \"Thu, 07 Mar 2024 22:48:58 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"1ceaa226-10b3-4e27-9531-44b7f32817a5\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:58.990827836Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:59.153652444Z","hostname":"buskin","pid":1661,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:54591"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:59.154202151Z","hostname":"buskin","pid":1661,"uri":"/inventory","method":"GET","req_id":"cc8da605-7716-467b-bb81-db7ce87514d4","remote_addr":"[fd00:1122:3344:101::b]:54591","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":514,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:59.154381906Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:59.154407787Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:59.154656644Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:59.15484599Z","hostname":"buskin","pid":1661,"uri":"/omicron-zones","method":"GET","req_id":"f348d313-93d9-4612-8ca0-7f5915ffd39c","remote_addr":"[fd00:1122:3344:101::b]:54591","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":485,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:48:59.244170408Z","hostname":"buskin","pid":1661,"uri":"/metrics/collect/cac4b398-c394-4c89-a5de-5577a51ce104","method":"GET","req_id":"6e9c8041-6e6c-495b-96ea-86f8ea581961","remote_addr":"[fd00:1122:3344:101::d]:39455","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":770,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:49:14.107189149Z","hostname":"buskin","pid":1661,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:54154"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:49:14.10724135Z","hostname":"buskin","pid":1661,"uri":"/baseboard","method":"GET","req_id":"86d48ed4-274a-457f-be0f-94ecbdff41de","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:54154","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":30,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:49:25.505003154Z","hostname":"buskin","pid":1661,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:34437"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:49:25.50519104Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0b3b3eae-a95f-4091-984a-93fccf167e79, kind: Service { id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c }, name: Name(\"external-dns-e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: ca9c5211-aa62-4394-a496-e20bf51739bc, kind: Service { id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b }, name: Name(\"external-dns-472c8e8c-ac8f-4193-921c-49dd3ceb288b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1eeeae19-992a-47f8-9e7b-dc93c05f0958, kind: Service { id: 761d7d83-4f44-4dad-ba8f-694aeae46e71 }, name: Name(\"nexus-761d7d83-4f44-4dad-ba8f-694aeae46e71\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 89eadaf9-099e-4929-ad88-e7ee2258e13d, kind: Service { id: 0a835148-a2f0-40d4-bc66-05f1d779465b }, name: Name(\"nexus-0a835148-a2f0-40d4-bc66-05f1d779465b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 72])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 8c296dc4-9b38-40c8-9549-80b2967f6cdc, kind: Service { id: 2bea071a-6ee2-45ba-9a69-93a0e2e5b31b }, name: Name(\"nexus-2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:49:25.505237261Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:49:25.505267052Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:49:25.505272342Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:49:25.505282342Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:49:25.505288623Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:49:25.505295443Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:49:25.505320393Z","hostname":"buskin","pid":1661,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"fe29e8d1-ac1d-42ed-811c-db00bacf6d5d","remote_addr":"[fd00:1122:3344:101::b]:34437","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":261,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5a90d047-3e3f-4448-8841-bd84373c9354\", \"content-length\": \"127\", \"date\": \"Thu, 07 Mar 2024 22:49:25 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"5a90d047-3e3f-4448-8841-bd84373c9354\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:49:25.567704007Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:49:25.699656942Z","hostname":"buskin","pid":1661,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:48169"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:49:25.699720214Z","hostname":"buskin","pid":1661,"uri":"/inventory","method":"GET","req_id":"8360b23b-69a2-41b6-bed8-6ae8bfdc18d4","remote_addr":"[fd00:1122:3344:101::b]:48169","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":41,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:49:25.700476027Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:49:25.700497868Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:49:25.700731055Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:49:25.70090446Z","hostname":"buskin","pid":1661,"uri":"/omicron-zones","method":"GET","req_id":"880a2604-9c1f-4b59-838e-cddd1c844cec","remote_addr":"[fd00:1122:3344:101::b]:48169","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1026,"response_code":"200"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:49:26.321604087Z","hostname":"buskin","pid":1661,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:776"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:49:29.243511474Z","hostname":"buskin","pid":1661,"uri":"/metrics/collect/cac4b398-c394-4c89-a5de-5577a51ce104","method":"GET","req_id":"bcd1e7ee-66fc-42b1-bf62-6aa289e26101","remote_addr":"[fd00:1122:3344:101::d]:39455","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":206,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:49:44.146050048Z","hostname":"buskin","pid":1661,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:43100"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:49:44.146092499Z","hostname":"buskin","pid":1661,"uri":"/baseboard","method":"GET","req_id":"e2bfd10a-40f7-4a8f-9a41-937dde5a8fef","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:43100","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":31,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:49:59.244182381Z","hostname":"buskin","pid":1661,"uri":"/metrics/collect/cac4b398-c394-4c89-a5de-5577a51ce104","method":"GET","req_id":"b4be8535-fea5-4de5-b825-1bc934d3a000","remote_addr":"[fd00:1122:3344:101::d]:39455","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":231,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:50:00.114832866Z","hostname":"buskin","pid":1661,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:41507"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:50:00.114980841Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0b3b3eae-a95f-4091-984a-93fccf167e79, kind: Service { id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c }, name: Name(\"external-dns-e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: ca9c5211-aa62-4394-a496-e20bf51739bc, kind: Service { id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b }, name: Name(\"external-dns-472c8e8c-ac8f-4193-921c-49dd3ceb288b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1eeeae19-992a-47f8-9e7b-dc93c05f0958, kind: Service { id: 761d7d83-4f44-4dad-ba8f-694aeae46e71 }, name: Name(\"nexus-761d7d83-4f44-4dad-ba8f-694aeae46e71\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 89eadaf9-099e-4929-ad88-e7ee2258e13d, kind: Service { id: 0a835148-a2f0-40d4-bc66-05f1d779465b }, name: Name(\"nexus-0a835148-a2f0-40d4-bc66-05f1d779465b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 72])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 8c296dc4-9b38-40c8-9549-80b2967f6cdc, kind: Service { id: 2bea071a-6ee2-45ba-9a69-93a0e2e5b31b }, name: Name(\"nexus-2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:50:00.115022402Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:50:00.115049353Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:50:00.115069253Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:50:00.115074924Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:50:00.115081784Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:50:00.115086924Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:50:00.115091964Z","hostname":"buskin","pid":1661,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"a3c7622b-7e4a-47f0-ba79-9fbc2706df02","remote_addr":"[fd00:1122:3344:101::b]:41507","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":226,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d0025375-5d88-4630-94d3-ea44ec8428a3\", \"content-length\": \"127\", \"date\": \"Thu, 07 Mar 2024 22:49:59 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"d0025375-5d88-4630-94d3-ea44ec8428a3\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:50:00.168675662Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:50:00.261453425Z","hostname":"buskin","pid":1661,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:61789"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:50:00.261558608Z","hostname":"buskin","pid":1661,"uri":"/inventory","method":"GET","req_id":"b664daeb-4a67-4e0c-b0d7-3f1e485e5168","remote_addr":"[fd00:1122:3344:101::b]:61789","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":36,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:50:00.261793265Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:50:00.261826676Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:50:00.262130515Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:50:00.262365992Z","hostname":"buskin","pid":1661,"uri":"/omicron-zones","method":"GET","req_id":"4218278d-d196-49e8-a4c3-cccc0c60688a","remote_addr":"[fd00:1122:3344:101::b]:61789","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":601,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:50:14.178968661Z","hostname":"buskin","pid":1661,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:57777"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:50:14.179022172Z","hostname":"buskin","pid":1661,"uri":"/baseboard","method":"GET","req_id":"38691711-6b8b-44b6-a59c-af27e5902cba","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:57777","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":29,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:50:29.245375766Z","hostname":"buskin","pid":1661,"uri":"/metrics/collect/cac4b398-c394-4c89-a5de-5577a51ce104","method":"GET","req_id":"28bc5244-a6ab-40ee-accd-78888ba9991f","remote_addr":"[fd00:1122:3344:101::d]:39455","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1013,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:50:37.560286892Z","hostname":"buskin","pid":1661,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:60833"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:50:37.560438156Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0b3b3eae-a95f-4091-984a-93fccf167e79, kind: Service { id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c }, name: Name(\"external-dns-e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: ca9c5211-aa62-4394-a496-e20bf51739bc, kind: Service { id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b }, name: Name(\"external-dns-472c8e8c-ac8f-4193-921c-49dd3ceb288b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1eeeae19-992a-47f8-9e7b-dc93c05f0958, kind: Service { id: 761d7d83-4f44-4dad-ba8f-694aeae46e71 }, name: Name(\"nexus-761d7d83-4f44-4dad-ba8f-694aeae46e71\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 89eadaf9-099e-4929-ad88-e7ee2258e13d, kind: Service { id: 0a835148-a2f0-40d4-bc66-05f1d779465b }, name: Name(\"nexus-0a835148-a2f0-40d4-bc66-05f1d779465b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 72])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 8c296dc4-9b38-40c8-9549-80b2967f6cdc, kind: Service { id: 2bea071a-6ee2-45ba-9a69-93a0e2e5b31b }, name: Name(\"nexus-2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:50:37.560478068Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:50:37.560501198Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:50:37.560508158Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:50:37.560520949Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:50:37.560531159Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:50:37.561105787Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:50:37.561131467Z","hostname":"buskin","pid":1661,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"82bb6425-5909-43d9-8218-9812efba3469","remote_addr":"[fd00:1122:3344:101::b]:60833","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":802,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"10d6dff3-36ae-45a5-8db6-ed3cc5eebc85\", \"content-length\": \"127\", \"date\": \"Thu, 07 Mar 2024 22:50:37 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"10d6dff3-36ae-45a5-8db6-ed3cc5eebc85\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:50:37.618969154Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:50:37.73172877Z","hostname":"buskin","pid":1661,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:63812"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:50:37.731807592Z","hostname":"buskin","pid":1661,"uri":"/inventory","method":"GET","req_id":"709d5077-96dd-4a90-8e03-24c4eb9e7d78","remote_addr":"[fd00:1122:3344:101::b]:63812","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":42,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:50:37.732528904Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:50:37.732550935Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:50:37.732783532Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:50:37.732974738Z","hostname":"buskin","pid":1661,"uri":"/omicron-zones","method":"GET","req_id":"fe0f8beb-8b34-490c-9628-3262a8ceca99","remote_addr":"[fd00:1122:3344:101::b]:63812","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":995,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:50:44.21275043Z","hostname":"buskin","pid":1661,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:57140"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:50:44.212824952Z","hostname":"buskin","pid":1661,"uri":"/baseboard","method":"GET","req_id":"906998cc-adf2-41d6-a3cf-c5519bc29d0b","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:57140","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":42,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:50:59.244632993Z","hostname":"buskin","pid":1661,"uri":"/metrics/collect/cac4b398-c394-4c89-a5de-5577a51ce104","method":"GET","req_id":"495a01c1-128a-48aa-81c2-f1c17fb95b1d","remote_addr":"[fd00:1122:3344:101::d]:39455","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":294,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:51:06.108826756Z","hostname":"buskin","pid":1661,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:60135"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:51:06.109029502Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0b3b3eae-a95f-4091-984a-93fccf167e79, kind: Service { id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c }, name: Name(\"external-dns-e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: ca9c5211-aa62-4394-a496-e20bf51739bc, kind: Service { id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b }, name: Name(\"external-dns-472c8e8c-ac8f-4193-921c-49dd3ceb288b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1eeeae19-992a-47f8-9e7b-dc93c05f0958, kind: Service { id: 761d7d83-4f44-4dad-ba8f-694aeae46e71 }, name: Name(\"nexus-761d7d83-4f44-4dad-ba8f-694aeae46e71\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 89eadaf9-099e-4929-ad88-e7ee2258e13d, kind: Service { id: 0a835148-a2f0-40d4-bc66-05f1d779465b }, name: Name(\"nexus-0a835148-a2f0-40d4-bc66-05f1d779465b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 72])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 8c296dc4-9b38-40c8-9549-80b2967f6cdc, kind: Service { id: 2bea071a-6ee2-45ba-9a69-93a0e2e5b31b }, name: Name(\"nexus-2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:51:06.109088464Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:51:06.109119265Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:51:06.109127175Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:51:06.109153486Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:51:06.109160526Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:51:06.109166496Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:51:06.109180487Z","hostname":"buskin","pid":1661,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"d64876a0-7ed2-429d-abc5-41ecb02e9bb3","remote_addr":"[fd00:1122:3344:101::b]:60135","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":323,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0576e2cc-8873-4f31-82ba-24be9de713b9\", \"content-length\": \"127\", \"date\": \"Thu, 07 Mar 2024 22:51:06 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"0576e2cc-8873-4f31-82ba-24be9de713b9\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:51:06.157880388Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:51:06.252276209Z","hostname":"buskin","pid":1661,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:40266"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:51:06.252335961Z","hostname":"buskin","pid":1661,"uri":"/inventory","method":"GET","req_id":"adf2e6d8-a16f-496e-9762-a1da9a2317fe","remote_addr":"[fd00:1122:3344:101::b]:40266","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":38,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:51:06.253112754Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:51:06.253138685Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:51:06.253377132Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:51:06.253551767Z","hostname":"buskin","pid":1661,"uri":"/omicron-zones","method":"GET","req_id":"bf7ac7de-8a3c-4ae6-a75d-25dc945075e8","remote_addr":"[fd00:1122:3344:101::b]:40266","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1085,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:51:14.247043827Z","hostname":"buskin","pid":1661,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:64777"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:51:14.247099849Z","hostname":"buskin","pid":1661,"uri":"/baseboard","method":"GET","req_id":"22172ead-e613-4013-b4fe-138788b44c17","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:64777","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":30,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:51:29.243611825Z","hostname":"buskin","pid":1661,"uri":"/metrics/collect/cac4b398-c394-4c89-a5de-5577a51ce104","method":"GET","req_id":"cc32a190-a661-4fc8-8653-31e21266e7f7","remote_addr":"[fd00:1122:3344:101::d]:39455","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":200,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:51:44.280373207Z","hostname":"buskin","pid":1661,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:54673"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:51:44.281100318Z","hostname":"buskin","pid":1661,"uri":"/baseboard","method":"GET","req_id":"34e9f490-6e94-4c4e-bcc1-419cad9e4b60","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:54673","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":703,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:51:55.963450249Z","hostname":"buskin","pid":1661,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:58466"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:51:55.963666596Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0b3b3eae-a95f-4091-984a-93fccf167e79, kind: Service { id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c }, name: Name(\"external-dns-e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: ca9c5211-aa62-4394-a496-e20bf51739bc, kind: Service { id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b }, name: Name(\"external-dns-472c8e8c-ac8f-4193-921c-49dd3ceb288b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1eeeae19-992a-47f8-9e7b-dc93c05f0958, kind: Service { id: 761d7d83-4f44-4dad-ba8f-694aeae46e71 }, name: Name(\"nexus-761d7d83-4f44-4dad-ba8f-694aeae46e71\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 89eadaf9-099e-4929-ad88-e7ee2258e13d, kind: Service { id: 0a835148-a2f0-40d4-bc66-05f1d779465b }, name: Name(\"nexus-0a835148-a2f0-40d4-bc66-05f1d779465b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 72])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 8c296dc4-9b38-40c8-9549-80b2967f6cdc, kind: Service { id: 2bea071a-6ee2-45ba-9a69-93a0e2e5b31b }, name: Name(\"nexus-2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:51:55.963697317Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:51:55.963703217Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:51:55.963707837Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:51:55.963712007Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:51:55.963716428Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:51:55.963720328Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:51:55.963730128Z","hostname":"buskin","pid":1661,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"d329b2dd-b214-44e9-8f3e-d1acae026160","remote_addr":"[fd00:1122:3344:101::b]:58466","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":224,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ace29850-21b9-4268-bae3-d19dda0af785\", \"content-length\": \"127\", \"date\": \"Thu, 07 Mar 2024 22:51:56 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"ace29850-21b9-4268-bae3-d19dda0af785\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:51:56.045224419Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:51:56.168363208Z","hostname":"buskin","pid":1661,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:39060"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:51:56.1690882Z","hostname":"buskin","pid":1661,"uri":"/inventory","method":"GET","req_id":"39e84d09-70c8-48f7-831b-ae3a0167323c","remote_addr":"[fd00:1122:3344:101::b]:39060","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":679,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:51:56.169299907Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:51:56.169324367Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:51:56.169562675Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:51:56.16974171Z","hostname":"buskin","pid":1661,"uri":"/omicron-zones","method":"GET","req_id":"465f4e91-337d-4565-aebc-bbd68a2083ba","remote_addr":"[fd00:1122:3344:101::b]:39060","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":462,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:51:59.244274188Z","hostname":"buskin","pid":1661,"uri":"/metrics/collect/cac4b398-c394-4c89-a5de-5577a51ce104","method":"GET","req_id":"6a76ec9b-2c02-4736-bf74-118f64ec07e9","remote_addr":"[fd00:1122:3344:101::d]:39455","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":213,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:52:14.315730309Z","hostname":"buskin","pid":1661,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:64280"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:52:14.315798041Z","hostname":"buskin","pid":1661,"uri":"/baseboard","method":"GET","req_id":"8ee30fdb-567c-441a-a22e-b0c1cba64309","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:64280","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":50,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:52:29.24350584Z","hostname":"buskin","pid":1661,"uri":"/metrics/collect/cac4b398-c394-4c89-a5de-5577a51ce104","method":"GET","req_id":"3b8daca9-b99d-44cb-b1e2-3d9358794acd","remote_addr":"[fd00:1122:3344:101::d]:39455","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":214,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:52:43.719768395Z","hostname":"buskin","pid":1661,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:50823"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:52:43.720693133Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0b3b3eae-a95f-4091-984a-93fccf167e79, kind: Service { id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c }, name: Name(\"external-dns-e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: ca9c5211-aa62-4394-a496-e20bf51739bc, kind: Service { id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b }, name: Name(\"external-dns-472c8e8c-ac8f-4193-921c-49dd3ceb288b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1eeeae19-992a-47f8-9e7b-dc93c05f0958, kind: Service { id: 761d7d83-4f44-4dad-ba8f-694aeae46e71 }, name: Name(\"nexus-761d7d83-4f44-4dad-ba8f-694aeae46e71\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 89eadaf9-099e-4929-ad88-e7ee2258e13d, kind: Service { id: 0a835148-a2f0-40d4-bc66-05f1d779465b }, name: Name(\"nexus-0a835148-a2f0-40d4-bc66-05f1d779465b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 72])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 8c296dc4-9b38-40c8-9549-80b2967f6cdc, kind: Service { id: 2bea071a-6ee2-45ba-9a69-93a0e2e5b31b }, name: Name(\"nexus-2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:52:43.720740004Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:52:43.720769895Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:52:43.720776965Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:52:43.720785516Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:52:43.720793156Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:52:43.720803496Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:52:43.720833627Z","hostname":"buskin","pid":1661,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"ea3ecfeb-4dde-4b19-86b2-cc2131b9a318","remote_addr":"[fd00:1122:3344:101::b]:50823","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1003,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"cd2cc492-fa1d-4252-84b7-74ae6089f128\", \"content-length\": \"127\", \"date\": \"Thu, 07 Mar 2024 22:52:43 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"cd2cc492-fa1d-4252-84b7-74ae6089f128\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:52:43.788288655Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:52:43.927776337Z","hostname":"buskin","pid":1661,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:37496"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:52:43.927842279Z","hostname":"buskin","pid":1661,"uri":"/inventory","method":"GET","req_id":"799a2726-ba01-4ab4-9815-ae93227d43c0","remote_addr":"[fd00:1122:3344:101::b]:37496","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":41,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:52:43.928032105Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:52:43.928057146Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:52:43.928308393Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:52:43.929694395Z","hostname":"buskin","pid":1661,"uri":"/omicron-zones","method":"GET","req_id":"dcdf627b-a7d3-4166-aa9e-210abfdaee3a","remote_addr":"[fd00:1122:3344:101::b]:37496","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1684,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:52:44.350116913Z","hostname":"buskin","pid":1661,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:49225"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:52:44.350194526Z","hostname":"buskin","pid":1661,"uri":"/baseboard","method":"GET","req_id":"6eeb2dd9-8abd-4114-87c7-5912f770e104","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:49225","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":43,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:52:59.243198696Z","hostname":"buskin","pid":1661,"uri":"/metrics/collect/cac4b398-c394-4c89-a5de-5577a51ce104","method":"GET","req_id":"5bf27993-ede4-4330-976a-7a78c5a14764","remote_addr":"[fd00:1122:3344:101::d]:39455","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":187,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:53:14.382129644Z","hostname":"buskin","pid":1661,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:46270"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:53:14.382188286Z","hostname":"buskin","pid":1661,"uri":"/baseboard","method":"GET","req_id":"835c9886-9ff6-4992-88e4-10da1eda14fc","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:46270","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":35,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:53:16.946354149Z","hostname":"buskin","pid":1661,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:45652"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:53:16.946521294Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0b3b3eae-a95f-4091-984a-93fccf167e79, kind: Service { id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c }, name: Name(\"external-dns-e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: ca9c5211-aa62-4394-a496-e20bf51739bc, kind: Service { id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b }, name: Name(\"external-dns-472c8e8c-ac8f-4193-921c-49dd3ceb288b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1eeeae19-992a-47f8-9e7b-dc93c05f0958, kind: Service { id: 761d7d83-4f44-4dad-ba8f-694aeae46e71 }, name: Name(\"nexus-761d7d83-4f44-4dad-ba8f-694aeae46e71\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 89eadaf9-099e-4929-ad88-e7ee2258e13d, kind: Service { id: 0a835148-a2f0-40d4-bc66-05f1d779465b }, name: Name(\"nexus-0a835148-a2f0-40d4-bc66-05f1d779465b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 72])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 8c296dc4-9b38-40c8-9549-80b2967f6cdc, kind: Service { id: 2bea071a-6ee2-45ba-9a69-93a0e2e5b31b }, name: Name(\"nexus-2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:53:16.946566626Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:53:16.946594507Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:53:16.946601587Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:53:16.946606967Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:53:16.946613947Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:53:16.946628988Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:53:16.946641658Z","hostname":"buskin","pid":1661,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"69971e3b-e8c5-4843-861d-fdd540f554d0","remote_addr":"[fd00:1122:3344:101::b]:45652","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":246,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c96ac42a-c9cd-44a8-a2c2-f1c75a3b32d5\", \"content-length\": \"127\", \"date\": \"Thu, 07 Mar 2024 22:53:16 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"c96ac42a-c9cd-44a8-a2c2-f1c75a3b32d5\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:53:17.008559218Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:53:17.131955095Z","hostname":"buskin","pid":1661,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:53940"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:53:17.132026907Z","hostname":"buskin","pid":1661,"uri":"/inventory","method":"GET","req_id":"7b5ed29d-616b-435e-a48e-f97af3c6e42c","remote_addr":"[fd00:1122:3344:101::b]:53940","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":35,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:53:17.132233434Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:53:17.132252774Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:53:17.132520322Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:53:17.132721388Z","hostname":"buskin","pid":1661,"uri":"/omicron-zones","method":"GET","req_id":"1767f5ce-2897-4f8c-97dd-3d775f2dd674","remote_addr":"[fd00:1122:3344:101::b]:53940","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":519,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:53:29.243220092Z","hostname":"buskin","pid":1661,"uri":"/metrics/collect/cac4b398-c394-4c89-a5de-5577a51ce104","method":"GET","req_id":"1b106485-f616-4a0a-8062-9b32df874212","remote_addr":"[fd00:1122:3344:101::d]:39455","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":228,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:53:37.482807795Z","hostname":"buskin","pid":1661,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:44120"} {"msg":"ensuring instance is registered","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:53:37.49223356Z","hostname":"buskin","pid":1661,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:505","metadata":"InstanceMetadata { silo_id: 1920a3e5-6255-40c6-98f1-d67a52b3718d, project_id: 674fe5c6-94d5-4069-b3e7-882343eb7675 }","propolis_addr":"[fd00:1122:3344:101::1:0]:12400","vmm_runtime":"VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-03-07T22:53:37.163221Z }","instance_runtime":"InstanceRuntimeState { propolis_id: Some(36152756-0bc4-4d23-aa5e-7b49326cdd50), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-03-07T22:53:37.186924118Z }","hardware":"InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: a620e8f5-605c-4231-a415-f7b75efea2f9, kind: Instance { id: 11545bc7-59aa-486a-980d-a53b8fa9a4e6 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 245, 20, 122])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(5016042), primary: true, slot: 0 }], source_nat: SourceNatConfig { ip: 10.151.1.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.1.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: a620e8f5-605c-4231-a415-f7b75efea2f9, kind: Instance { id: 11545bc7-59aa-486a-980d-a53b8fa9a4e6 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 245, 20, 122])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(5016042), primary: true, slot: 0 }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: a620e8f5-605c-4231-a415-f7b75efea2f9, kind: Instance { id: 11545bc7-59aa-486a-980d-a53b8fa9a4e6 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 245, 20, 122])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(5016042), primary: true, slot: 0 }], filter_hosts: Some([Vpc(Vni(5016042))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: a620e8f5-605c-4231-a415-f7b75efea2f9, kind: Instance { id: 11545bc7-59aa-486a-980d-a53b8fa9a4e6 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 245, 20, 122])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(5016042), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-c826eb34aa5\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: e0896dc8-cf06-4e95-aedd-6519cee42d73, read_only_parent: Some(Volume { block_size: 512, id: d3d7f21a-e901-4b6c-837b-3a9485d4337c, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 64f61506-2d1e-4935-9c3e-232e76347abc, key: Some(\"HcdWvJq2BiNa4y/JD2fxBFmbcNkzpn5SUF6v++jwwQg=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::1a]:19001\", \"[fd00:1122:3344:101::19]:19001\", \"[fd00:1122:3344:101::13]:19002\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: e0896dc8-cf06-4e95-aedd-6519cee42d73, key: Some(\"t8+nRK6RKJesGOvaL1BpDWEdPK/qExY3eO83C4JcNGI=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::19]:19002\", \"[fd00:1122:3344:101::16]:19001\", \"[fd00:1122:3344:101::12]:19000\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAACQsrZnWGdYAACytmdYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAkLK2Z1hnWAAAsrZnWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiIxMTU0NWJjNy01OWFhLTQ4NmEtOTgwZC1hNTNiOGZhOWE0ZTYiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJQ0EwMUxySlZ1U0hxUEhKbmJwMmlhNW0wQ3ZzUGxTdGJ2MGRPbCtNcGJ3MCJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }","propolis_id":"36152756-0bc4-4d23-aa5e-7b49326cdd50","instance_id":"11545bc7-59aa-486a-980d-a53b8fa9a4e6"} {"msg":"registering new instance","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:53:37.492808737Z","hostname":"buskin","pid":1661,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:539","instance_id":"11545bc7-59aa-486a-980d-a53b8fa9a4e6"} {"msg":"initializing new Instance","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:53:37.494114987Z","hostname":"buskin","pid":1661,"component":"InstanceManager","file":"sled-agent/src/instance.rs:938","state":"InstanceInitialState { hardware: InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: a620e8f5-605c-4231-a415-f7b75efea2f9, kind: Instance { id: 11545bc7-59aa-486a-980d-a53b8fa9a4e6 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 245, 20, 122])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(5016042), primary: true, slot: 0 }], source_nat: SourceNatConfig { ip: 10.151.1.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.1.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: a620e8f5-605c-4231-a415-f7b75efea2f9, kind: Instance { id: 11545bc7-59aa-486a-980d-a53b8fa9a4e6 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 245, 20, 122])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(5016042), primary: true, slot: 0 }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: a620e8f5-605c-4231-a415-f7b75efea2f9, kind: Instance { id: 11545bc7-59aa-486a-980d-a53b8fa9a4e6 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 245, 20, 122])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(5016042), primary: true, slot: 0 }], filter_hosts: Some([Vpc(Vni(5016042))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: a620e8f5-605c-4231-a415-f7b75efea2f9, kind: Instance { id: 11545bc7-59aa-486a-980d-a53b8fa9a4e6 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 245, 20, 122])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(5016042), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-c826eb34aa5\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: e0896dc8-cf06-4e95-aedd-6519cee42d73, read_only_parent: Some(Volume { block_size: 512, id: d3d7f21a-e901-4b6c-837b-3a9485d4337c, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 64f61506-2d1e-4935-9c3e-232e76347abc, key: Some(\"HcdWvJq2BiNa4y/JD2fxBFmbcNkzpn5SUF6v++jwwQg=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::1a]:19001\", \"[fd00:1122:3344:101::19]:19001\", \"[fd00:1122:3344:101::13]:19002\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: e0896dc8-cf06-4e95-aedd-6519cee42d73, key: Some(\"t8+nRK6RKJesGOvaL1BpDWEdPK/qExY3eO83C4JcNGI=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::19]:19002\", \"[fd00:1122:3344:101::16]:19001\", \"[fd00:1122:3344:101::12]:19000\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAACQsrZnWGdYAACytmdYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAkLK2Z1hnWAAAsrZnWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiIxMTU0NWJjNy01OWFhLTQ4NmEtOTgwZC1hNTNiOGZhOWE0ZTYiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJQ0EwMUxySlZ1U0hxUEhKbmJwMmlhNW0wQ3ZzUGxTdGJ2MGRPbCtNcGJ3MCJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }, instance_runtime: InstanceRuntimeState { propolis_id: Some(36152756-0bc4-4d23-aa5e-7b49326cdd50), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-03-07T22:53:37.186924118Z }, vmm_runtime: VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-03-07T22:53:37.163221Z }, propolis_addr: [fd00:1122:3344:101::1:0]:12400 }","propolis_id":"36152756-0bc4-4d23-aa5e-7b49326cdd50","instance_id":"11545bc7-59aa-486a-980d-a53b8fa9a4e6"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:53:37.495410456Z","hostname":"buskin","pid":1661,"uri":"/instances/11545bc7-59aa-486a-980d-a53b8fa9a4e6","method":"PUT","req_id":"795d9859-a6f3-47ed-96a0-3618b07bc960","remote_addr":"[fd00:1122:3344:101::a]:44120","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":12551,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:53:37.592023183Z","hostname":"buskin","pid":1661,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:39560"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:53:37.618262276Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte6\", ip: 172.30.0.5, mac: MacAddr6([168, 64, 37, 245, 20, 122]), slot: 0, vni: Vni { inner: 5016042 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.0.1 }, vnic: \"vopte6\" } }"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:53:37.618390929Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Configuring new Omicron zone: oxz_propolis-server_36152756-0bc4-4d23-aa5e-7b49326cdd50","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:53:37.653369176Z","hostname":"buskin","pid":1661,"instance_id":"11545bc7-59aa-486a-980d-a53b8fa9a4e6","component":"InstanceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_propolis-server_36152756-0bc4-4d23-aa5e-7b49326cdd50","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:53:37.674581566Z","hostname":"buskin","pid":1661,"instance_id":"11545bc7-59aa-486a-980d-a53b8fa9a4e6","component":"InstanceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_propolis-server_36152756-0bc4-4d23-aa5e-7b49326cdd50:\n\n\n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:53:39.587110269Z","hostname":"buskin","pid":1661,"instance_id":"11545bc7-59aa-486a-980d-a53b8fa9a4e6","component":"InstanceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:53:39.587356116Z","hostname":"buskin","pid":1661,"zone":"oxz_propolis-server_36152756-0bc4-4d23-aa5e-7b49326cdd50","instance_id":"11545bc7-59aa-486a-980d-a53b8fa9a4e6","component":"InstanceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"running automatic periodic zone bundle cleanup","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:53:39.701951857Z","hostname":"buskin","pid":1661,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:299"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:53:39.7020241Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:53:39.738226483Z","hostname":"buskin","pid":1661,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:288","time_to_next_cleanup":"599.99999765s","next_cleanup":"Instant { tv_sec: 2279, tv_nsec: 855149023 }"} {"msg":"wait for service Some(\"oxz_propolis-server_36152756-0bc4-4d23-aa5e-7b49326cdd50\") failed: Property not found. retry in 40.462812ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:53:39.786537392Z","hostname":"buskin","pid":1661,"zone":"oxz_propolis-server_36152756-0bc4-4d23-aa5e-7b49326cdd50","instance_id":"11545bc7-59aa-486a-980d-a53b8fa9a4e6","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_36152756-0bc4-4d23-aa5e-7b49326cdd50\") failed: Property not found. retry in 140.013651ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:53:39.835035287Z","hostname":"buskin","pid":1661,"zone":"oxz_propolis-server_36152756-0bc4-4d23-aa5e-7b49326cdd50","instance_id":"11545bc7-59aa-486a-980d-a53b8fa9a4e6","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_36152756-0bc4-4d23-aa5e-7b49326cdd50\") failed: Property not found. retry in 114.273454ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:53:39.987472091Z","hostname":"buskin","pid":1661,"zone":"oxz_propolis-server_36152756-0bc4-4d23-aa5e-7b49326cdd50","instance_id":"11545bc7-59aa-486a-980d-a53b8fa9a4e6","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_36152756-0bc4-4d23-aa5e-7b49326cdd50\") failed: Property not found. retry in 363.380722ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:53:40.118963822Z","hostname":"buskin","pid":1661,"zone":"oxz_propolis-server_36152756-0bc4-4d23-aa5e-7b49326cdd50","instance_id":"11545bc7-59aa-486a-980d-a53b8fa9a4e6","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_36152756-0bc4-4d23-aa5e-7b49326cdd50\") failed: Property not found. retry in 730.255142ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:53:40.496287219Z","hostname":"buskin","pid":1661,"zone":"oxz_propolis-server_36152756-0bc4-4d23-aa5e-7b49326cdd50","instance_id":"11545bc7-59aa-486a-980d-a53b8fa9a4e6","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_36152756-0bc4-4d23-aa5e-7b49326cdd50\") failed: Property not found. retry in 991.900331ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:53:41.238513785Z","hostname":"buskin","pid":1661,"zone":"oxz_propolis-server_36152756-0bc4-4d23-aa5e-7b49326cdd50","instance_id":"11545bc7-59aa-486a-980d-a53b8fa9a4e6","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_36152756-0bc4-4d23-aa5e-7b49326cdd50\") failed: Property not found. retry in 1.409958702s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:53:42.241781526Z","hostname":"buskin","pid":1661,"zone":"oxz_propolis-server_36152756-0bc4-4d23-aa5e-7b49326cdd50","instance_id":"11545bc7-59aa-486a-980d-a53b8fa9a4e6","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_36152756-0bc4-4d23-aa5e-7b49326cdd50\") failed: Property not found. retry in 1.093315621s","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:53:43.663216206Z","hostname":"buskin","pid":1661,"zone":"oxz_propolis-server_36152756-0bc4-4d23-aa5e-7b49326cdd50","instance_id":"11545bc7-59aa-486a-980d-a53b8fa9a4e6","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:53:44.414539878Z","hostname":"buskin","pid":1661,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:53021"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:53:44.414756525Z","hostname":"buskin","pid":1661,"uri":"/baseboard","method":"GET","req_id":"acce8255-0c21-434b-9acd-959b2122225e","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:53021","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":126,"response_code":"200"} {"msg":"Started propolis in zone: oxz_propolis-server_36152756-0bc4-4d23-aa5e-7b49326cdd50","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:53:44.789841593Z","hostname":"buskin","pid":1661,"instance_id":"11545bc7-59aa-486a-980d-a53b8fa9a4e6","component":"InstanceManager","file":"sled-agent/src/instance.rs:1425"} {"msg":"wait for service Some(\"oxz_propolis-server_36152756-0bc4-4d23-aa5e-7b49326cdd50\") failed: Property not found. retry in 45.500593ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:53:44.799453413Z","hostname":"buskin","pid":1661,"instance_id":"11545bc7-59aa-486a-980d-a53b8fa9a4e6","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_36152756-0bc4-4d23-aa5e-7b49326cdd50\") failed: Property not found. retry in 132.722023ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:53:44.855517686Z","hostname":"buskin","pid":1661,"instance_id":"11545bc7-59aa-486a-980d-a53b8fa9a4e6","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_36152756-0bc4-4d23-aa5e-7b49326cdd50\") failed: Property not found. retry in 201.934754ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:53:44.997190415Z","hostname":"buskin","pid":1661,"instance_id":"11545bc7-59aa-486a-980d-a53b8fa9a4e6","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_36152756-0bc4-4d23-aa5e-7b49326cdd50\") failed: Property not found. retry in 566.547083ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:53:45.210180068Z","hostname":"buskin","pid":1661,"instance_id":"11545bc7-59aa-486a-980d-a53b8fa9a4e6","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Propolis SMF service is online","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:53:45.787532325Z","hostname":"buskin","pid":1661,"instance_id":"11545bc7-59aa-486a-980d-a53b8fa9a4e6","component":"InstanceManager","file":"sled-agent/src/instance.rs:1434"} {"msg":"Propolis HTTP server online","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:53:45.824730518Z","hostname":"buskin","pid":1661,"instance_id":"11545bc7-59aa-486a-980d-a53b8fa9a4e6","component":"InstanceManager","file":"sled-agent/src/instance.rs:1448"} {"msg":"Sending ensure request to propolis: InstanceEnsureRequest { cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAACQsrZnWGdYAACytmdYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAkLK2Z1hnWAAAsrZnWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiIxMTU0NWJjNy01OWFhLTQ4NmEtOTgwZC1hNTNiOGZhOWE0ZTYiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJQ0EwMUxySlZ1U0hxUEhKbmJwMmlhNW0wQ3ZzUGxTdGJ2MGRPbCtNcGJ3MCJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\"), disks: [DiskRequest { device: \"nvme\", name: \"disk-c826eb34aa5\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: e0896dc8-cf06-4e95-aedd-6519cee42d73, read_only_parent: Some(Volume { block_size: 512, id: d3d7f21a-e901-4b6c-837b-3a9485d4337c, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 64f61506-2d1e-4935-9c3e-232e76347abc, key: Some(\"HcdWvJq2BiNa4y/JD2fxBFmbcNkzpn5SUF6v++jwwQg=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::1a]:19001\", \"[fd00:1122:3344:101::19]:19001\", \"[fd00:1122:3344:101::13]:19002\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: e0896dc8-cf06-4e95-aedd-6519cee42d73, key: Some(\"t8+nRK6RKJesGOvaL1BpDWEdPK/qExY3eO83C4JcNGI=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::19]:19002\", \"[fd00:1122:3344:101::16]:19001\", \"[fd00:1122:3344:101::12]:19000\"] } }] } }], migrate: None, nics: [NetworkInterfaceRequest { name: \"vopte6\", slot: Slot(0) }], properties: InstanceProperties { bootrom_id: 00000000-0000-0000-0000-000000000000, description: \"Test description\", id: 11545bc7-59aa-486a-980d-a53b8fa9a4e6, image_id: 00000000-0000-0000-0000-000000000000, memory: 1024, name: \"localshark\", vcpus: 2 } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:53:45.828204503Z","hostname":"buskin","pid":1661,"instance_id":"11545bc7-59aa-486a-980d-a53b8fa9a4e6","component":"InstanceManager","file":"sled-agent/src/instance.rs:686"} {"msg":"result of instance_ensure call is Ok(InstanceEnsureResponse { migrate: None })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:53:45.977044719Z","hostname":"buskin","pid":1661,"instance_id":"11545bc7-59aa-486a-980d-a53b8fa9a4e6","component":"InstanceManager","file":"sled-agent/src/instance.rs:688"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:53:45.97740421Z","hostname":"buskin","pid":1661,"uri":"/instances/11545bc7-59aa-486a-980d-a53b8fa9a4e6/state","method":"PUT","req_id":"4b134b95-2231-4667-8462-6d64e70cc876","remote_addr":"[fd00:1122:3344:101::a]:39560","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":8385388,"response_code":"200"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Starting), migration_status: NoMigration, time: 2024-03-07T22:53:45.977458381Z }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:53:45.978037569Z","hostname":"buskin","pid":1661,"instance_id":"11545bc7-59aa-486a-980d-a53b8fa9a4e6","component":"InstanceManager","file":"sled-agent/src/instance.rs:570"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:53:45.978221174Z","hostname":"buskin","pid":1661,"instance_id":"11545bc7-59aa-486a-980d-a53b8fa9a4e6","component":"InstanceManager","file":"sled-agent/src/instance.rs:592","new_vmm_state":"VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-03-07T22:53:45.977458381Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(36152756-0bc4-4d23-aa5e-7b49326cdd50), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-03-07T22:53:37.186924118Z }","propolis_id":"36152756-0bc4-4d23-aa5e-7b49326cdd50"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:53:45.978239195Z","hostname":"buskin","pid":1661,"instance_id":"11545bc7-59aa-486a-980d-a53b8fa9a4e6","component":"InstanceManager","file":"sled-agent/src/instance.rs:492","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(36152756-0bc4-4d23-aa5e-7b49326cdd50), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-03-07T22:53:37.186924118Z }, propolis_id: 36152756-0bc4-4d23-aa5e-7b49326cdd50, vmm_state: VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-03-07T22:53:45.977458381Z } }","instance_id":"11545bc7-59aa-486a-980d-a53b8fa9a4e6"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Running), migration_status: NoMigration, time: 2024-03-07T22:53:46.026596835Z }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:53:46.026609256Z","hostname":"buskin","pid":1661,"instance_id":"11545bc7-59aa-486a-980d-a53b8fa9a4e6","component":"InstanceManager","file":"sled-agent/src/instance.rs:570"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:53:46.026621886Z","hostname":"buskin","pid":1661,"instance_id":"11545bc7-59aa-486a-980d-a53b8fa9a4e6","component":"InstanceManager","file":"sled-agent/src/instance.rs:592","new_vmm_state":"VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-03-07T22:53:46.026596835Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(36152756-0bc4-4d23-aa5e-7b49326cdd50), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-03-07T22:53:37.186924118Z }","propolis_id":"36152756-0bc4-4d23-aa5e-7b49326cdd50"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:53:46.026628286Z","hostname":"buskin","pid":1661,"instance_id":"11545bc7-59aa-486a-980d-a53b8fa9a4e6","component":"InstanceManager","file":"sled-agent/src/instance.rs:492","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(36152756-0bc4-4d23-aa5e-7b49326cdd50), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-03-07T22:53:37.186924118Z }, propolis_id: 36152756-0bc4-4d23-aa5e-7b49326cdd50, vmm_state: VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-03-07T22:53:46.026596835Z } }","instance_id":"11545bc7-59aa-486a-980d-a53b8fa9a4e6"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:53:59.244351597Z","hostname":"buskin","pid":1661,"uri":"/metrics/collect/cac4b398-c394-4c89-a5de-5577a51ce104","method":"GET","req_id":"72816ac5-c636-4a59-adaf-9b2a376d98e2","remote_addr":"[fd00:1122:3344:101::d]:39455","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1377,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:54:02.900119508Z","hostname":"buskin","pid":1661,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:40737"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:54:02.900536211Z","hostname":"buskin","pid":1661,"uri":"/instances/11545bc7-59aa-486a-980d-a53b8fa9a4e6/state","method":"PUT","req_id":"0c1fbffc-767b-4fcb-aafe-a316cdb1a42e","remote_addr":"[fd00:1122:3344:101::a]:40737","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":350,"response_code":"200"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Stopping), migration_status: NoMigration, time: 2024-03-07T22:54:02.900599223Z }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:54:02.900612473Z","hostname":"buskin","pid":1661,"instance_id":"11545bc7-59aa-486a-980d-a53b8fa9a4e6","component":"InstanceManager","file":"sled-agent/src/instance.rs:570"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:54:02.900619563Z","hostname":"buskin","pid":1661,"instance_id":"11545bc7-59aa-486a-980d-a53b8fa9a4e6","component":"InstanceManager","file":"sled-agent/src/instance.rs:592","new_vmm_state":"VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-03-07T22:54:02.900599223Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(36152756-0bc4-4d23-aa5e-7b49326cdd50), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-03-07T22:53:37.186924118Z }","propolis_id":"36152756-0bc4-4d23-aa5e-7b49326cdd50"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:54:02.900625724Z","hostname":"buskin","pid":1661,"instance_id":"11545bc7-59aa-486a-980d-a53b8fa9a4e6","component":"InstanceManager","file":"sled-agent/src/instance.rs:492","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(36152756-0bc4-4d23-aa5e-7b49326cdd50), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-03-07T22:53:37.186924118Z }, propolis_id: 36152756-0bc4-4d23-aa5e-7b49326cdd50, vmm_state: VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-03-07T22:54:02.900599223Z } }","instance_id":"11545bc7-59aa-486a-980d-a53b8fa9a4e6"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Destroyed), migration_status: NoMigration, time: 2024-03-07T22:54:02.922410562Z }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:54:02.922423762Z","hostname":"buskin","pid":1661,"instance_id":"11545bc7-59aa-486a-980d-a53b8fa9a4e6","component":"InstanceManager","file":"sled-agent/src/instance.rs:570"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:54:02.922444183Z","hostname":"buskin","pid":1661,"instance_id":"11545bc7-59aa-486a-980d-a53b8fa9a4e6","component":"InstanceManager","file":"sled-agent/src/instance.rs:592","new_vmm_state":"VmmRuntimeState { state: Destroyed, gen: Generation(6), time_updated: 2024-03-07T22:54:02.922410562Z }","new_instance_state":"InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-03-07T22:54:02.922410562Z }","propolis_id":"36152756-0bc4-4d23-aa5e-7b49326cdd50"} {"msg":"terminating VMM that has exited","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:54:02.922452613Z","hostname":"buskin","pid":1661,"instance_id":"11545bc7-59aa-486a-980d-a53b8fa9a4e6","component":"InstanceManager","file":"sled-agent/src/instance.rs:607","instance_id":"11545bc7-59aa-486a-980d-a53b8fa9a4e6"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:54:02.922457863Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:54:02.922622758Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"creating zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:54:02.922642869Z","hostname":"buskin","pid":1661,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:436","context":"ZoneBundleContext { storage_dirs: [\"/pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/debug/bundle/zone\", \"/pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/debug/bundle/zone\"], cause: TerminatedInstance, extra_log_dirs: [\"/pool/ext/14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/debug\", \"/pool/ext/24b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/debug\", \"/pool/ext/31bd71cd-4736-4a12-a387-9b74b050396f/crypt/debug\", \"/pool/ext/616b26df-e62a-4c68-b506-f4a923d8aaf7/crypt/debug\", \"/pool/ext/cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crypt/debug\", \"/pool/ext/ceb4461c-cf56-4719-ad3c-14430bfdfb60/crypt/debug\", \"/pool/ext/d462a7f7-b628-40fe-80ff-4e4189e2d62b/crypt/debug\", \"/pool/ext/e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/debug\", \"/pool/ext/f4b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/debug\"] }","zone_name":"oxz_propolis-server_36152756-0bc4-4d23-aa5e-7b49326cdd50"} {"msg":"finished zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:54:03.515288258Z","hostname":"buskin","pid":1661,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:1235","metadata":"ZoneBundleMetadata { id: ZoneBundleId { zone_name: \"oxz_propolis-server_36152756-0bc4-4d23-aa5e-7b49326cdd50\", bundle_id: f17a2342-b9dc-42f5-9c71-ab6248623e6b }, time_created: 2024-03-07T22:54:02.922895726Z, version: 0, cause: TerminatedInstance }"} {"msg":"Halting and removing zone: oxz_propolis-server_36152756-0bc4-4d23-aa5e-7b49326cdd50","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:54:03.51535913Z","hostname":"buskin","pid":1661,"instance_id":"11545bc7-59aa-486a-980d-a53b8fa9a4e6","component":"InstanceManager","file":"sled-agent/src/instance.rs:781"} {"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:54:05.296505244Z","hostname":"buskin","pid":1661,"instance_id":"11545bc7-59aa-486a-980d-a53b8fa9a4e6","component":"InstanceManager","file":"illumos-utils/src/zone.rs:269"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:54:05.324582932Z","hostname":"buskin","pid":1661,"instance_id":"11545bc7-59aa-486a-980d-a53b8fa9a4e6","component":"InstanceManager","file":"sled-agent/src/instance.rs:492","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-03-07T22:54:02.922410562Z }, propolis_id: 36152756-0bc4-4d23-aa5e-7b49326cdd50, vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-03-07T22:54:05.324516050Z } }","instance_id":"11545bc7-59aa-486a-980d-a53b8fa9a4e6"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:54:05.402802434Z","hostname":"buskin","pid":1661,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:48764"} {"msg":"unset_virtual_nic_host unimplmented","v":0,"name":"SledAgent","level":40,"time":"2024-03-07T22:54:05.403179516Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:620"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:54:05.403202337Z","hostname":"buskin","pid":1661,"uri":"/v2p/a620e8f5-605c-4231-a415-f7b75efea2f9","method":"DELETE","req_id":"c3c58856-5dbb-46a8-9c1b-627a76ccaa42","remote_addr":"[fd00:1122:3344:101::b]:48764","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":287,"response_code":"204"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:54:05.519706765Z","hostname":"buskin","pid":1661,"instance_id":"11545bc7-59aa-486a-980d-a53b8fa9a4e6","component":"InstanceManager","file":"sled-agent/src/instance.rs:492","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-03-07T22:54:02.922410562Z }, propolis_id: 36152756-0bc4-4d23-aa5e-7b49326cdd50, vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-03-07T22:54:05.324516050Z } }","instance_id":"11545bc7-59aa-486a-980d-a53b8fa9a4e6"} {"msg":"State monitoring task complete","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:54:05.523470079Z","hostname":"buskin","pid":1661,"instance_id":"11545bc7-59aa-486a-980d-a53b8fa9a4e6","component":"InstanceManager","file":"sled-agent/src/instance.rs:724"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:54:09.422527268Z","hostname":"buskin","pid":1661,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:50067"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:54:09.422781835Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0b3b3eae-a95f-4091-984a-93fccf167e79, kind: Service { id: e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c }, name: Name(\"external-dns-e3cbbfb4-7a0a-4cd8-9919-13da60e29b8c\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: ca9c5211-aa62-4394-a496-e20bf51739bc, kind: Service { id: 472c8e8c-ac8f-4193-921c-49dd3ceb288b }, name: Name(\"external-dns-472c8e8c-ac8f-4193-921c-49dd3ceb288b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 87])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1eeeae19-992a-47f8-9e7b-dc93c05f0958, kind: Service { id: 761d7d83-4f44-4dad-ba8f-694aeae46e71 }, name: Name(\"nexus-761d7d83-4f44-4dad-ba8f-694aeae46e71\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 172])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 89eadaf9-099e-4929-ad88-e7ee2258e13d, kind: Service { id: 0a835148-a2f0-40d4-bc66-05f1d779465b }, name: Name(\"nexus-0a835148-a2f0-40d4-bc66-05f1d779465b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 72])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 8c296dc4-9b38-40c8-9549-80b2967f6cdc, kind: Service { id: 2bea071a-6ee2-45ba-9a69-93a0e2e5b31b }, name: Name(\"nexus-2bea071a-6ee2-45ba-9a69-93a0e2e5b31b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 48])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:54:09.422840007Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:54:09.422890819Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:54:09.422902799Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:54:09.422911029Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:54:09.42292444Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:54:09.42293622Z","hostname":"buskin","pid":1661,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:54:09.422973961Z","hostname":"buskin","pid":1661,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"e93455a4-8edc-4c08-828d-6eb274211b33","remote_addr":"[fd00:1122:3344:101::b]:50067","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":374,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a8f1d1d9-9894-42a6-847e-99893fed8353\", \"content-length\": \"127\", \"date\": \"Thu, 07 Mar 2024 22:54:09 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"a8f1d1d9-9894-42a6-847e-99893fed8353\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:54:09.47657686Z","hostname":"buskin","pid":1661,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:54:09.566583838Z","hostname":"buskin","pid":1661,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:44016"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:54:09.56666191Z","hostname":"buskin","pid":1661,"uri":"/inventory","method":"GET","req_id":"81ce896c-1c9c-46ed-b493-523760f2af30","remote_addr":"[fd00:1122:3344:101::b]:44016","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":48,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:54:09.567552467Z","hostname":"buskin","pid":1661,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:54:09.56766258Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:54:09.56796894Z","hostname":"buskin","pid":1661,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-07T22:54:09.568247628Z","hostname":"buskin","pid":1661,"uri":"/omicron-zones","method":"GET","req_id":"c5a7bb96-b5f9-4d37-b248-949eb78142f6","remote_addr":"[fd00:1122:3344:101::b]:44016","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1438,"response_code":"200"}