[ Feb 26 07:11:17 Enabled. ] [ Feb 26 07:11:17 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ] [ Feb 26 07:11:17 Method "start" exited with status 0. ] [ Feb 26 07:11:17 Rereading configuration. ] [ Feb 26 07:11:17 No 'refresh' method defined. Treating as :true. ] note: configured to log to "/dev/stdout" {"msg":"finding links [\"net0\", \"net1\"]","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:17.872904978Z","hostname":"buskin","pid":1671,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:247"} {"msg":"Starting mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:17.925817211Z","hostname":"buskin","pid":1671,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:254","addr-objs":"[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]"} {"msg":"Importing mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:17.925846331Z","hostname":"buskin","pid":1671,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:42","path":"/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"} {"msg":"Setting mg-ddm interfaces","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:18.03757548Z","hostname":"buskin","pid":1671,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:50","interfaces":"(\"net0/ll\" \"net1/ll\")"} {"msg":"Enabling mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:18.046576189Z","hostname":"buskin","pid":1671,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:56"} {"msg":"Ensuring zfs key directory exists","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:18.05459696Z","hostname":"buskin","pid":1671,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:264","path":"/var/run/oxide/"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:18.658643508Z","hostname":"buskin","pid":1671,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:91","prefix":"Ipv6Prefix { addr: fdb0:18c0:4d0d:9fb2::, len: 64 }"} {"msg":"Starting KeyManager","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:18.796790283Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:117"} {"msg":"Starting StorageManager","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:18.796823823Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:129"} {"msg":"Starting StorageMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:18.796910945Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:141"} {"msg":"Starting HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:18.797009326Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:162","sled_mode":"Scrimlet { asic: SoftNpuZone }"} {"msg":"DumpSetup poll loop started.","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:18.797023377Z","hostname":"buskin","pid":1671,"component":"DumpSetup-archival","file":"sled-agent/src/dump_setup.rs:268"} {"msg":"Creating HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:18.797032967Z","hostname":"buskin","pid":1671,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:609"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:18.797058547Z","hostname":"buskin","pid":1671,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:776"} {"msg":"Generated i86pc baseboard Pc { identifier: \"buskin\", model: \"i86pc\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:18.799905738Z","hostname":"buskin","pid":1671,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:535"} {"msg":"Device is not a Gimlet (i86pc), proceeding with null hardware view","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:11:18.799919788Z","hostname":"buskin","pid":1671,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:658"} {"msg":"Starting HardwareMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:18.799924388Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:176"} {"msg":"Monitoring for hardware updates","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:18.799928758Z","hostname":"buskin","pid":1671,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:716"} {"msg":"Upserting synthetic zpool to Storage Manager: oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:18.799940498Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:18.799955408Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:18.799962288Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:18.799966368Z","hostname":"buskin","pid":1671,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:241","underlay_network_info":"None"} {"msg":"Upserting synthetic zpool to Storage Manager: oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:18.799974619Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:18.799978679Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:18.799983639Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:18.799987509Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:18.799993889Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:18.799997889Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxp_31bd71cd-4736-4a12-a387-9b74b050396f","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:18.800006729Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:18.800010839Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Waiting for boot disk","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:18.800014759Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:88"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: a462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal } }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:18.800021329Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Channel capacity decreased","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:18.983046366Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:326","current":"245","previous":"256"} {"msg":"Found boot disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:18.983090356Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:90"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: b462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal } }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:18.983097546Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.156502423Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.156551114Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.156564484Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.156569484Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.156575244Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received DisksChanged({})","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.156579364Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.156616465Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.156630335Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.156635325Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.156641385Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Channel capacity increased","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.156645515Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:336","current":"255","previous":"245"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.156658666Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.156731597Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:111","resources":"StorageResources { disks: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: a462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal } }), Pool { name: ZpoolName { id: a462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal }, info: ZpoolInfo { name: \"oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b\", size: 20937965568, allocated: 497664, free: 20937467904, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: b462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal } }), Pool { name: ZpoolName { id: b462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal }, info: ZpoolInfo { name: \"oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b\", size: 20937965568, allocated: 543744, free: 20937421824, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" } })} }"} {"msg":"Starting Bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.156769988Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:200"} {"msg":"No ledger in /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.156857099Z","hostname":"buskin","pid":1671,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:187"} {"msg":"No ledger in /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.156878539Z","hostname":"buskin","pid":1671,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:187"} {"msg":"No ledger found. Loading Fsm::State as Uninitialized","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.15689332Z","hostname":"buskin","pid":1671,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:83"} {"msg":"No ledger in /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.15691375Z","hostname":"buskin","pid":1671,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:187"} {"msg":"No ledger in /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.15692647Z","hostname":"buskin","pid":1671,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:187"} {"msg":"No ledger found for network config","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.15693134Z","hostname":"buskin","pid":1671,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:146"} {"msg":"Starting Bootstore DDMD poller","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.156965761Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:205"} {"msg":"Starting ZoneBundler related tasks","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.156973351Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:220"} {"msg":"Creating ServiceManager","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.171220912Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:700"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.171250793Z","hostname":"buskin","pid":1671,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:288","time_to_next_cleanup":"599.999954509s","next_cleanup":"Instant { tv_sec: 90652, tv_nsec: 376118389 }"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.171274943Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3407"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.171310204Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"No ledger in /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.171365485Z","hostname":"buskin","pid":1671,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:187"} {"msg":"No ledger in /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.173010504Z","hostname":"buskin","pid":1671,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:187"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.174084843Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:195"} {"msg":"Started listening","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.174173984Z","hostname":"buskin","pid":1671,"component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:44","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Injecting RSS configuration: RackInitializeRequest {\n trust_quorum_peers: None,\n bootstrap_discovery: OnlyOurs,\n ntp_servers: [\n \"0.pool.ntp.org\",\n ],\n dns_servers: [\n 1.1.1.1,\n 9.9.9.9,\n ],\n internal_services_ip_pool_ranges: [\n V4(\n Ipv4Range {\n first: 10.151.1.164,\n last: 10.151.1.173,\n },\n ),\n ],\n external_dns_ips: [\n 10.151.1.164,\n 10.151.1.165,\n ],\n external_dns_zone_name: \"oxide.test\",\n external_certificates: \"\",\n recovery_silo: RecoverySiloConfig {\n silo_name: Name(\n \"recovery\",\n ),\n user_name: UserId(\n \"recovery\",\n ),\n user_password_hash: NewPasswordHash(\n PasswordHashString {\n string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n encoding: B64,\n },\n ),\n },\n rack_network_config: RackNetworkConfigV1 {\n rack_subnet: Ipv6Network {\n addr: fd00:1122:3344:100::,\n prefix: 56,\n },\n infra_ip_first: 192.168.1.30,\n infra_ip_last: 192.168.1.30,\n ports: [\n PortConfigV1 {\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Network {\n addr: 0.0.0.0,\n prefix: 0,\n },\n ),\n nexthop: 10.151.1.100,\n },\n ],\n addresses: [\n V4(\n Ipv4Network {\n addr: 10.151.1.174,\n prefix: 24,\n },\n ),\n ],\n switch: Switch0,\n port: \"qsfp0\",\n uplink_port_speed: Speed40G,\n uplink_port_fec: None,\n bgp_peers: [],\n autoneg: false,\n },\n ],\n bgp: [],\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.17449798Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:800"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.17451468Z","hostname":"buskin","pid":1671,"component":"DnsResolver","component":"RSS","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.174594792Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"No ledger in /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.174644603Z","hostname":"buskin","pid":1671,"component":"RSS","file":"common/src/ledger.rs:187"} {"msg":"No ledger in /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.174664903Z","hostname":"buskin","pid":1671,"component":"RSS","file":"common/src/ledger.rs:187"} {"msg":"RSS configuration has not been fully applied yet","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.174672183Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:860"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.174694253Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"No ledger in /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.174709014Z","hostname":"buskin","pid":1671,"component":"RSS","file":"common/src/ledger.rs:187"} {"msg":"No ledger in /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.174721194Z","hostname":"buskin","pid":1671,"component":"RSS","file":"common/src/ledger.rs:187"} {"msg":"Creating new allocation plan","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.174740814Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:901"} {"msg":"Serializing plan","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.174745264Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:118"} {"msg":"Creating plan for the sled at fdb0:18c0:4d0d:9fb2::1","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.174749265Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:91"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.174780455Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.174797635Z","hostname":"buskin","pid":1671,"component":"RSS","file":"common/src/ledger.rs:198"} {"msg":"Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.175474737Z","hostname":"buskin","pid":1671,"component":"RSS","file":"common/src/ledger.rs:198"} {"msg":"Sled plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.175607009Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:138"} {"msg":"Writing Rack Network Configuration to bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.17561534Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:934"} {"msg":"Attempting to update network config with generation: 1, current_generation: 0","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.17564875Z","hostname":"buskin","pid":1671,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.175662941Z","hostname":"buskin","pid":1671,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:198"} {"msg":"Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.175785153Z","hostname":"buskin","pid":1671,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:198"} {"msg":"Broadcasting network config with generation 1","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.175921435Z","hostname":"buskin","pid":1671,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"Received initialization request from RSS","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.175980356Z","hostname":"buskin","pid":1671,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:217","target_sled":"[fdb0:18c0:4d0d:9fb2::1]:12346","request":"StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 5abda959-c9da-461c-83f2-8406a5d578e7, rack_id: ce675376-0228-48ba-a7e8-cc3953e7c762, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net(Ipv6Network { addr: fd00:1122:3344:101::, prefix: 64 }) } } }"} {"msg":"Accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.176126659Z","hostname":"buskin","pid":1671,"remote_addr":"[fdb0:18c0:4d0d:9fb2::1]:36088","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:66"} {"msg":"Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 5abda959-c9da-461c-83f2-8406a5d578e7, rack_id: ce675376-0228-48ba-a7e8-cc3953e7c762, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net(Ipv6Network { addr: fd00:1122:3344:101::, prefix: 64 }) } } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.17622534Z","hostname":"buskin","pid":1671,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:364"} {"msg":"KeyManager: using hardcoded secret retriever","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.176231931Z","hostname":"buskin","pid":1671,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:379"} {"msg":"setting up sled agent server","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.176256991Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/server.rs:47"} {"msg":"Received KeyManagerReady","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.176267001Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Attempting to add queued disks","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.176270481Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:394","num_disks":"9"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.176274231Z","hostname":"buskin","pid":1671,"component":"DnsResolver","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"Enabling switch zone (new)","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.178334398Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3635"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.184845892Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:327","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.184924204Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:329","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.184930404Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:333","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.184961474Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:335","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.185033046Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:37"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.185037796Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:347"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.215789408Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:56"} {"msg":"Automatically destroying dataset: oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.223254029Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:389"} {"msg":"SledAgent::new(..) starting","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.293862334Z","hostname":"buskin","pid":1671,"sled_id":"5abda959-c9da-461c-83f2-8406a5d578e7","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:342"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.293902585Z","hostname":"buskin","pid":1671,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:91","prefix":"Ipv6Prefix { addr: fd00:1122:3344:101::, len: 64 }"} {"msg":"Looking for unencrypted datasets in oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.328908272Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:474"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.353632447Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:327","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.353694399Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:329","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.353710569Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:333","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.35378174Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:335","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.353907792Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:37"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.353916923Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:347"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.381786124Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:56"} {"msg":"Automatically destroying dataset: oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.389670323Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:389"} {"msg":"Looking for unencrypted datasets in oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.496049398Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:474"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.519029273Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:327","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\",\n model: \"synthetic-model\",\n}"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.519072363Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:329","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.519089214Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:333","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.519102064Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:335","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.519234186Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:37"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.519244877Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:347"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.547080707Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:56"} {"msg":"Automatically destroying dataset: oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.555088048Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:389"} {"msg":"Looking for unencrypted datasets in oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.661124577Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:474"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.68456524Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:327","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.684613832Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:329","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.684637122Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:333","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.684679402Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:335","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.684812095Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:37"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.684820865Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:347"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.712849439Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:56"} {"msg":"Automatically destroying dataset: oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.720400562Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:389"} {"msg":"Looking for unencrypted datasets in oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.825890052Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:474"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.849171772Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:327","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\",\n model: \"synthetic-model\",\n}"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.849220363Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:329","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.849231583Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:333","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.849255683Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:335","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.849355305Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:37"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.849366485Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:347"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.876965852Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:56"} {"msg":"Automatically destroying dataset: oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.885486762Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:389"} {"msg":"Looking for unencrypted datasets in oxp_31bd71cd-4736-4a12-a387-9b74b050396f","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:19.988977176Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:474"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:20.012167935Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:327","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\",\n model: \"synthetic-model\",\n}"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:20.012230596Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:329","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:20.012240296Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:333","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:20.012252406Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:335","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:20.012371278Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:37"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:20.012380489Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:347"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:20.04082554Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:56"} {"msg":"Automatically destroying dataset: oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:20.04880034Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:389"} {"msg":"Looking for unencrypted datasets in oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:20.15711728Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:474"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:20.179794379Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:327","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\",\n model: \"synthetic-model\",\n}"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:20.179887341Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:329","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:20.179897091Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:333","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:20.179938512Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:335","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:20.180061524Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:37"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:20.180067764Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:347"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:20.208942243Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:56"} {"msg":"Automatically destroying dataset: oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:20.216636069Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:389"} {"msg":"Looking for unencrypted datasets in oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:20.322899852Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:474"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:20.345857696Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:327","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\",\n model: \"synthetic-model\",\n}"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:20.345919368Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:329","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:20.345927968Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:333","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:20.345935878Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:335","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:20.34607048Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:37"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:20.34607851Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:347"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:20.373602705Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:56"} {"msg":"Automatically destroying dataset: oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:20.381348522Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:389"} {"msg":"Looking for unencrypted datasets in oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:20.486256451Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:474"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:20.508816879Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:327","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:20.50886738Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:329","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:20.50887416Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:333","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:20.50891629Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:335","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:20.509044313Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:37"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:20.509050333Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:347"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:20.536561418Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:56"} {"msg":"Automatically destroying dataset: oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:20.544114181Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:389"} {"msg":"Looking for unencrypted datasets in oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:20.648498911Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:474"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:20.664072075Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:20.664121976Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Requested swap device of size 64 GiB","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:20.664136377Z","hostname":"buskin","pid":1671,"sled_id":"5abda959-c9da-461c-83f2-8406a5d578e7","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:354"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:20.669474931Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:111","resources":"StorageResources { disks: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }), Pool { name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, info: ZpoolInfo { name: \"oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03\", size: 20937965568, allocated: 777216, free: 20937188352, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }), Pool { name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, info: ZpoolInfo { name: \"oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03\", size: 20937965568, allocated: 766464, free: 20937199104, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }), Pool { name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, info: ZpoolInfo { name: \"oxp_31bd71cd-4736-4a12-a387-9b74b050396f\", size: 20937965568, allocated: 774144, free: 20937191424, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } }), Pool { name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, info: ZpoolInfo { name: \"oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7\", size: 20937965568, allocated: 772608, free: 20937192960, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: a462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal } }), Pool { name: ZpoolName { id: a462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal }, info: ZpoolInfo { name: \"oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b\", size: 20937965568, allocated: 497664, free: 20937467904, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: b462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal } }), Pool { name: ZpoolName { id: b462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal }, info: ZpoolInfo { name: \"oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b\", size: 20937965568, allocated: 543744, free: 20937421824, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } }), Pool { name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, info: ZpoolInfo { name: \"oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", size: 20937965568, allocated: 772608, free: 20937192960, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } }), Pool { name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, info: ZpoolInfo { name: \"oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60\", size: 20937965568, allocated: 1735680, free: 20936229888, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } }), Pool { name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, info: ZpoolInfo { name: \"oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b\", size: 20937965568, allocated: 774144, free: 20937191424, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }), Pool { name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, info: ZpoolInfo { name: \"oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", size: 20937965568, allocated: 771072, free: 20937194496, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }), Pool { name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, info: ZpoolInfo { name: \"oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", size: 20937965568, allocated: 777216, free: 20937188352, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" } })} }"} {"msg":"attempting to create encrypted zvol: name=\"oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:20.687502728Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/swap_device.rs:153"} {"msg":"Configuring new Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:20.68983513Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:20.708291065Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"successfully created encrypted zvol: name=\"oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:20.71028384Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/swap_device.rs:249"} {"msg":"adding swap device: swapname=\"/dev/zvol/dsk/oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b/swap\"","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:20.710440853Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/swap_device.rs:96"} {"msg":"Mounting backing filesystems","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:20.715637944Z","hostname":"buskin","pid":1671,"sled_id":"5abda959-c9da-461c-83f2-8406a5d578e7","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:369"} {"msg":"Processing fmd","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:20.715649874Z","hostname":"buskin","pid":1671,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:126"} {"msg":"Ensuring dataset oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b/backing/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:20.715655074Z","hostname":"buskin","pid":1671,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:136"} {"msg":"Stopping service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:20.779385128Z","hostname":"buskin","pid":1671,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:170"} {"msg":"Mounting oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b/backing/fmd on /var/fm/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:20.998800965Z","hostname":"buskin","pid":1671,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:178"} {"msg":"Ensuring directory /var/fm/fmd/rsrc","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:21.011490709Z","hostname":"buskin","pid":1671,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Ensuring directory /var/fm/fmd/ckpt","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:21.01158212Z","hostname":"buskin","pid":1671,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Ensuring directory /var/fm/fmd/xprt","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:21.011615841Z","hostname":"buskin","pid":1671,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Starting service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:21.011631791Z","hostname":"buskin","pid":1671,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:192"} {"msg":"Ensuring contract reaper thread","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:21.181713669Z","hostname":"buskin","pid":1671,"file":"illumos-utils/src/running_zone.rs:163"} {"msg":"using '[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]' as data links for xde driver","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:21.72297594Z","hostname":"buskin","pid":1671,"sled_id":"5abda959-c9da-461c-83f2-8406a5d578e7","component":"SledAgent","file":"illumos-utils/src/opte/illumos.rs:90"} {"msg":"Underlay Available","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:21.723530129Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:122","sled_id":"5abda959-c9da-461c-83f2-8406a5d578e7"} {"msg":"50% of 102952026112 physical ram = 51476013056 bytes)","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:21.72358073Z","hostname":"buskin","pid":1671,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:206"} {"msg":"Setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:21.723589231Z","hostname":"buskin","pid":1671,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:214"} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: 5abda959-c9da-461c-83f2-8406a5d578e7, variant: M2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:21.737926383Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: 5abda959-c9da-461c-83f2-8406a5d578e7, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:21.738235578Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: 5abda959-c9da-461c-83f2-8406a5d578e7, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:21.738365921Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:21.738566455Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/5abda959-c9da-461c-83f2-8406a5d578e7/zpools/24b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:21.739014692Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/5abda959-c9da-461c-83f2-8406a5d578e7/zpools/cd70d7f6-2354-4bf2-8012-55bf9eaf7930): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: 5abda959-c9da-461c-83f2-8406a5d578e7, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:21.739374579Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: 5abda959-c9da-461c-83f2-8406a5d578e7, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:21.739548052Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", sled_id: 5abda959-c9da-461c-83f2-8406a5d578e7, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:21.739643843Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", sled_id: 5abda959-c9da-461c-83f2-8406a5d578e7, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:21.739793396Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: 5abda959-c9da-461c-83f2-8406a5d578e7, variant: M2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:21.739905408Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", sled_id: 5abda959-c9da-461c-83f2-8406a5d578e7, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:21.74004955Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", sled_id: 5abda959-c9da-461c-83f2-8406a5d578e7, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:21.740191183Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: 5abda959-c9da-461c-83f2-8406a5d578e7, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:21.740326746Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:21.740492768Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/5abda959-c9da-461c-83f2-8406a5d578e7/zpools/14b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxp_31bd71cd-4736-4a12-a387-9b74b050396f on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:21.740643831Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/5abda959-c9da-461c-83f2-8406a5d578e7/zpools/31bd71cd-4736-4a12-a387-9b74b050396f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:21.740798554Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/5abda959-c9da-461c-83f2-8406a5d578e7/zpools/616b26df-e62a-4c68-b506-f4a923d8aaf7): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:21.740981447Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/5abda959-c9da-461c-83f2-8406a5d578e7/zpools/a462a7f7-b628-40fe-80ff-4e4189e2d62b): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:21.74112424Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/5abda959-c9da-461c-83f2-8406a5d578e7/zpools/b462a7f7-b628-40fe-80ff-4e4189e2d62b): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:21.741250792Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/5abda959-c9da-461c-83f2-8406a5d578e7/zpools/ceb4461c-cf56-4719-ad3c-14430bfdfb60): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:21.741370874Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/5abda959-c9da-461c-83f2-8406a5d578e7/zpools/d462a7f7-b628-40fe-80ff-4e4189e2d62b): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:21.741489586Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/5abda959-c9da-461c-83f2-8406a5d578e7/zpools/e4b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:21.741589768Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/5abda959-c9da-461c-83f2-8406a5d578e7/zpools/f4b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:24.611059625Z","hostname":"buskin","pid":1671,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 64.664065ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:11:24.887240073Z","hostname":"buskin","pid":1671,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 148.127269ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:11:25.104302609Z","hostname":"buskin","pid":1671,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 111.576459ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:11:25.297496704Z","hostname":"buskin","pid":1671,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 530.304132ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:11:25.51966993Z","hostname":"buskin","pid":1671,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 546.34546ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:11:26.164860982Z","hostname":"buskin","pid":1671,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.484709584s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:11:26.786223914Z","hostname":"buskin","pid":1671,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 870.264077ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:11:28.457535833Z","hostname":"buskin","pid":1671,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.404806561s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:11:29.559034088Z","hostname":"buskin","pid":1671,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"sled agent started","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:32.030116403Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:981","underlay_address":"fd00:1122:3344:101::1"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:32.030217855Z","hostname":"buskin","pid":1671,"component":"DnsResolver","component":"ServiceManager","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:32.087733129Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:195"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:32.087768159Z","hostname":"buskin","pid":1671,"sled_id":"5abda959-c9da-461c-83f2-8406a5d578e7","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"5abda959-c9da-461c-83f2-8406a5d578e7"} {"msg":"Sled Agent loaded; recording configuration","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:32.087781079Z","hostname":"buskin","pid":1671,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:424"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:32.08781862Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:32.087847161Z","hostname":"buskin","pid":1671,"component":"SledAgentMain","file":"common/src/ledger.rs:198"} {"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:11:32.087898712Z","hostname":"buskin","pid":1671,"sled_id":"5abda959-c9da-461c-83f2-8406a5d578e7","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1199","retry_after":"259.598713ms","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:32.088121925Z","hostname":"buskin","pid":1671,"component":"SledAgentMain","file":"common/src/ledger.rs:198"} {"msg":"Sled Agent Started","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:32.088300499Z","hostname":"buskin","pid":1671,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:137"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:32.088312999Z","hostname":"buskin","pid":1671,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:241","underlay_network_info":"Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV1 { rack_subnet: Ipv6Network { addr: fd00:1122:3344:100::, prefix: 56 }, infra_ip_first: 192.168.1.30, infra_ip_last: 192.168.1.30, ports: [PortConfigV1 { routes: [RouteConfig { destination: V4(Ipv4Network { addr: 0.0.0.0, prefix: 0 }), nexthop: 10.151.1.100 }], addresses: [V4(Ipv4Network { addr: 10.151.1.174, prefix: 24 })], switch: Switch0, port: \"qsfp0\", uplink_port_speed: Speed40G, uplink_port_fec: None, bgp_peers: [], autoneg: false }], bgp: [] })))"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:32.088321099Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3407"} {"msg":"Connection closed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:32.08835657Z","hostname":"buskin","pid":1671,"remote_addr":"[fdb0:18c0:4d0d:9fb2::1]:36088","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:73"} {"msg":"Peer agent initialized","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:32.091172999Z","hostname":"buskin","pid":1671,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:108","peer":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Initialized sled agent","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:32.091236971Z","hostname":"buskin","pid":1671,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:232","target_sled":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:32.091537326Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"No ledger in /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/rss-service-plan-v2.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:32.091661848Z","hostname":"buskin","pid":1671,"component":"RSS","file":"common/src/ledger.rs:187"} {"msg":"No ledger in /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/rss-service-plan-v2.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:32.091698159Z","hostname":"buskin","pid":1671,"component":"RSS","file":"common/src/ledger.rs:187"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:32.091706699Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"failed to notify nexus about sled agent","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:32.092167247Z","hostname":"buskin","pid":1671,"sled_id":"5abda959-c9da-461c-83f2-8406a5d578e7","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:735","error":"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/5abda959-c9da-461c-83f2-8406a5d578e7): error trying to connect: dns error: proto error: io error: No route to host (os error 148)"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:32.123715433Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:38129"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:32.123875756Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:32.123926237Z","hostname":"buskin","pid":1671,"uri":"/zpools","method":"GET","req_id":"9099d7d4-3696-4b28-b810-ae8fbb188195","remote_addr":"[fd00:1122:3344:101::1]:38129","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":123,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:32.152676853Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:64966"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:32.152819716Z","hostname":"buskin","pid":1671,"uri":"/sled-role","method":"GET","req_id":"4d330bbf-6d3a-4a8f-b38d-254c4d108dae","remote_addr":"[fd00:1122:3344:101::1]:64966","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":87,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:32.154871222Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.rss-service-plan-v2.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:32.154882572Z","hostname":"buskin","pid":1671,"component":"RSS","file":"common/src/ledger.rs:198"} {"msg":"Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.rss-service-plan-v2.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:32.155153997Z","hostname":"buskin","pid":1671,"component":"RSS","file":"common/src/ledger.rs:198"} {"msg":"Service plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:32.15532779Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/plan/service.rs:741"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:32.183522707Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:32.18368404Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:62173"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:32.183910644Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:32.183923464Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:32.187996116Z","hostname":"buskin","pid":1671,"sled_id":"5abda959-c9da-461c-83f2-8406a5d578e7","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"5abda959-c9da-461c-83f2-8406a5d578e7"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:32.236838537Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:32.236892578Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:32.287946838Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:32.290180547Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:32.316305938Z","hostname":"buskin","pid":1671,"sled_id":"5abda959-c9da-461c-83f2-8406a5d578e7","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"5abda959-c9da-461c-83f2-8406a5d578e7"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:32.339355994Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"No ledger in /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:32.339410765Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:187"} {"msg":"No ledger in /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:32.339417945Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:187"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:32.339460666Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:11:32.351307024Z","hostname":"buskin","pid":1671,"sled_id":"5abda959-c9da-461c-83f2-8406a5d578e7","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1199","retry_after":"330.520287ms","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:32.38507488Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:32.385140741Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:32.385145571Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_2a4f4adb-368e-4ad8-9ca4-08f3408cecf7","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:32.435812304Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_d7ff77b4-c6c4-4720-908c-7a0571f1e748","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:32.439809584Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_70de9569-4ef0-4038-88e6-16ed7c5649a0","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:32.443342797Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_internal_dns_2a4f4adb-368e-4ad8-9ca4-08f3408cecf7","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:32.455960109Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_d7ff77b4-c6c4-4720-908c-7a0571f1e748","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:32.46000341Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_70de9569-4ef0-4038-88e6-16ed7c5649a0","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:32.46394598Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:32.489867927Z","hostname":"buskin","pid":1671,"sled_id":"5abda959-c9da-461c-83f2-8406a5d578e7","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"5abda959-c9da-461c-83f2-8406a5d578e7"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:32.639362022Z","hostname":"buskin","pid":1671,"sled_id":"5abda959-c9da-461c-83f2-8406a5d578e7","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"5abda959-c9da-461c-83f2-8406a5d578e7"} {"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:11:32.684121431Z","hostname":"buskin","pid":1671,"sled_id":"5abda959-c9da-461c-83f2-8406a5d578e7","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1199","retry_after":"984.389862ms","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:32.856498649Z","hostname":"buskin","pid":1671,"sled_id":"5abda959-c9da-461c-83f2-8406a5d578e7","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"5abda959-c9da-461c-83f2-8406a5d578e7"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:33.145598254Z","hostname":"buskin","pid":1671,"sled_id":"5abda959-c9da-461c-83f2-8406a5d578e7","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"5abda959-c9da-461c-83f2-8406a5d578e7"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:33.371445095Z","hostname":"buskin","pid":1671,"sled_id":"5abda959-c9da-461c-83f2-8406a5d578e7","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"5abda959-c9da-461c-83f2-8406a5d578e7"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:33.637676358Z","hostname":"buskin","pid":1671,"sled_id":"5abda959-c9da-461c-83f2-8406a5d578e7","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"5abda959-c9da-461c-83f2-8406a5d578e7"} {"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:11:33.669796754Z","hostname":"buskin","pid":1671,"sled_id":"5abda959-c9da-461c-83f2-8406a5d578e7","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1199","retry_after":"2.400026529s","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"Ensuring tfportrear0_0/ll exists in zone","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:33.965184251Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1949"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:33.970286251Z","hostname":"buskin","pid":1671,"sled_id":"5abda959-c9da-461c-83f2-8406a5d578e7","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"5abda959-c9da-461c-83f2-8406a5d578e7"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:34.163522297Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_2a4f4adb-368e-4ad8-9ca4-08f3408cecf7","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:34.296202155Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_70de9569-4ef0-4038-88e6-16ed7c5649a0","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_internal_dns_2a4f4adb-368e-4ad8-9ca4-08f3408cecf7\") failed: Property not found. retry in 42.71308ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:11:34.308076425Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_2a4f4adb-368e-4ad8-9ca4-08f3408cecf7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:34.308185016Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_d7ff77b4-c6c4-4720-908c-7a0571f1e748","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_internal_dns_2a4f4adb-368e-4ad8-9ca4-08f3408cecf7\") failed: Property not found. retry in 147.591167ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:11:34.359060223Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_2a4f4adb-368e-4ad8-9ca4-08f3408cecf7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_70de9569-4ef0-4038-88e6-16ed7c5649a0\") failed: Property not found. retry in 58.638156ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:11:34.533236013Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_70de9569-4ef0-4038-88e6-16ed7c5649a0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_2a4f4adb-368e-4ad8-9ca4-08f3408cecf7\") failed: Property not found. retry in 272.098154ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:11:34.548758377Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_2a4f4adb-368e-4ad8-9ca4-08f3408cecf7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:34.663442988Z","hostname":"buskin","pid":1671,"sled_id":"5abda959-c9da-461c-83f2-8406a5d578e7","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"5abda959-c9da-461c-83f2-8406a5d578e7"} {"msg":"wait for service Some(\"oxz_internal_dns_d7ff77b4-c6c4-4720-908c-7a0571f1e748\") failed: Property not found. retry in 52.311378ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:11:34.66352933Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_d7ff77b4-c6c4-4720-908c-7a0571f1e748","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_70de9569-4ef0-4038-88e6-16ed7c5649a0\") failed: Property not found. retry in 121.41ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:11:34.81163399Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_70de9569-4ef0-4038-88e6-16ed7c5649a0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_d7ff77b4-c6c4-4720-908c-7a0571f1e748\") failed: Property not found. retry in 75.335469ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:11:34.820883443Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_d7ff77b4-c6c4-4720-908c-7a0571f1e748","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_2a4f4adb-368e-4ad8-9ca4-08f3408cecf7\") failed: Property not found. retry in 570.601278ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:11:34.839006113Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_2a4f4adb-368e-4ad8-9ca4-08f3408cecf7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_d7ff77b4-c6c4-4720-908c-7a0571f1e748\") failed: Property not found. retry in 103.184937ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:11:34.908493448Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_d7ff77b4-c6c4-4720-908c-7a0571f1e748","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_70de9569-4ef0-4038-88e6-16ed7c5649a0\") failed: Property not found. retry in 278.718046ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:11:34.923714526Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_70de9569-4ef0-4038-88e6-16ed7c5649a0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_d7ff77b4-c6c4-4720-908c-7a0571f1e748\") failed: Property not found. retry in 301.367691ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:11:35.07998311Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_d7ff77b4-c6c4-4720-908c-7a0571f1e748","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_70de9569-4ef0-4038-88e6-16ed7c5649a0\") failed: Property not found. retry in 416.888206ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:11:35.213840009Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_70de9569-4ef0-4038-88e6-16ed7c5649a0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_d7ff77b4-c6c4-4720-908c-7a0571f1e748\") failed: Property not found. retry in 919.099908ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:11:35.391885928Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_d7ff77b4-c6c4-4720-908c-7a0571f1e748","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_2a4f4adb-368e-4ad8-9ca4-08f3408cecf7\") failed: Property not found. retry in 1.153265821s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:11:35.419150208Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_2a4f4adb-368e-4ad8-9ca4-08f3408cecf7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:35.438255195Z","hostname":"buskin","pid":1671,"sled_id":"5abda959-c9da-461c-83f2-8406a5d578e7","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"5abda959-c9da-461c-83f2-8406a5d578e7"} {"msg":"wait for service Some(\"oxz_internal_dns_70de9569-4ef0-4038-88e6-16ed7c5649a0\") failed: Property not found. retry in 811.377415ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:11:35.64277039Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_70de9569-4ef0-4038-88e6-16ed7c5649a0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:11:36.071346984Z","hostname":"buskin","pid":1671,"sled_id":"5abda959-c9da-461c-83f2-8406a5d578e7","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1199","retry_after":"2.908679305s","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"wait for service Some(\"oxz_internal_dns_d7ff77b4-c6c4-4720-908c-7a0571f1e748\") failed: Property not found. retry in 643.495511ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:11:36.321941581Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_d7ff77b4-c6c4-4720-908c-7a0571f1e748","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_70de9569-4ef0-4038-88e6-16ed7c5649a0\") failed: Property not found. retry in 1.343071826s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:11:36.466219644Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_70de9569-4ef0-4038-88e6-16ed7c5649a0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:36.530157431Z","hostname":"buskin","pid":1671,"sled_id":"5abda959-c9da-461c-83f2-8406a5d578e7","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"5abda959-c9da-461c-83f2-8406a5d578e7"} {"msg":"wait for service Some(\"oxz_internal_dns_2a4f4adb-368e-4ad8-9ca4-08f3408cecf7\") failed: Property not found. retry in 980.083613ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:11:36.583003682Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_2a4f4adb-368e-4ad8-9ca4-08f3408cecf7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_d7ff77b4-c6c4-4720-908c-7a0571f1e748\") failed: Property not found. retry in 1.00981498s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:11:36.974991341Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_d7ff77b4-c6c4-4720-908c-7a0571f1e748","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:37.494283125Z","hostname":"buskin","pid":1671,"sled_id":"5abda959-c9da-461c-83f2-8406a5d578e7","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"5abda959-c9da-461c-83f2-8406a5d578e7"} {"msg":"wait for service Some(\"oxz_internal_dns_2a4f4adb-368e-4ad8-9ca4-08f3408cecf7\") failed: Property not found. retry in 1.105574116s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:11:37.573219656Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_2a4f4adb-368e-4ad8-9ca4-08f3408cecf7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_70de9569-4ef0-4038-88e6-16ed7c5649a0\") failed: Property not found. retry in 608.308613ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:11:37.817498371Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_70de9569-4ef0-4038-88e6-16ed7c5649a0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_d7ff77b4-c6c4-4720-908c-7a0571f1e748\") failed: Property not found. retry in 699.052393ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:11:37.992982895Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_d7ff77b4-c6c4-4720-908c-7a0571f1e748","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:38.238731876Z","hostname":"buskin","pid":1671,"sled_id":"5abda959-c9da-461c-83f2-8406a5d578e7","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"5abda959-c9da-461c-83f2-8406a5d578e7"} {"msg":"wait for service Some(\"oxz_internal_dns_70de9569-4ef0-4038-88e6-16ed7c5649a0\") failed: Property not found. retry in 1.161647908s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:11:38.434413485Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_70de9569-4ef0-4038-88e6-16ed7c5649a0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_2a4f4adb-368e-4ad8-9ca4-08f3408cecf7\") failed: Property not found. retry in 880.814582ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:11:38.688049066Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_2a4f4adb-368e-4ad8-9ca4-08f3408cecf7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_d7ff77b4-c6c4-4720-908c-7a0571f1e748\") failed: Property not found. retry in 718.175993ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:11:38.701183118Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_d7ff77b4-c6c4-4720-908c-7a0571f1e748","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:11:38.981475938Z","hostname":"buskin","pid":1671,"sled_id":"5abda959-c9da-461c-83f2-8406a5d578e7","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1199","retry_after":"11.045732025s","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:39.189848531Z","hostname":"buskin","pid":1671,"sled_id":"5abda959-c9da-461c-83f2-8406a5d578e7","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"5abda959-c9da-461c-83f2-8406a5d578e7"} {"msg":"Ensuring address fd00:1122:3344:3::1 exists","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:39.472242578Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2002"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:3::1, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:39.472291199Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_d7ff77b4-c6c4-4720-908c-7a0571f1e748","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"Ensuring bootstrap address fdb0:18c0:4d0d:9fb2::2 exists in switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:40.014521746Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1966"} {"msg":"Adding bootstrap address","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:40.014576557Z","hostname":"buskin","pid":1671,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:626"} {"msg":"Ensuring address fd00:1122:3344:3::1 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:40.030411516Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2010"} {"msg":"Zone not using gateway (even though underlay is up)","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:40.030465887Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2034"} {"msg":"Setting up internal-dns service","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:40.207890144Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2187"} {"msg":"Forwarding bootstrap traffic via oxBootstrap0 to fe80::8:20ff:fe11:1ca","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:40.56503895Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1973"} {"msg":"Zone not using gateway (even though underlay is up)","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:40.570283512Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2034"} {"msg":"Setting up dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:40.689449243Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2466"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:40.737807025Z","hostname":"buskin","pid":1671,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:91","prefix":"Ipv6Prefix { addr: fd00:1122:3344:3::, len: 64 }"} {"msg":"Ensuring address fd00:1122:3344:1::1 exists","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:40.788902556Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2002"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:1::1, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:40.788941166Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_2a4f4adb-368e-4ad8-9ca4-08f3408cecf7","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"Setting up MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:40.943245396Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2345"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:41.032271565Z","hostname":"buskin","pid":1671,"sled_id":"5abda959-c9da-461c-83f2-8406a5d578e7","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"5abda959-c9da-461c-83f2-8406a5d578e7"} {"msg":"Setting up wicketd service","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:41.233758957Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2380"} {"msg":"Ensuring address fd00:1122:3344:1::1 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:41.337922793Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2010"} {"msg":"Zone not using gateway (even though underlay is up)","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:41.337956023Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2034"} {"msg":"Setting up mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:41.433408166Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2647"} {"msg":"Setting up internal-dns service","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:41.514011936Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2187"} {"msg":"Setting up mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:41.577847122Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2652"} {"msg":"Setting up tfport service","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:41.758288262Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2586"} {"msg":"Setting up Simulated SP service","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:41.926242563Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2374"} {"msg":"Re-enabling running switch zone (new address)","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:41.935876892Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3659","new":"[fd00:1122:3344:101::2, ::1]","old":"[::1]"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:41.935915083Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3675"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:41.935924363Z","hostname":"buskin","pid":1671,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"Ensuring address fd00:1122:3344:2::1 exists","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:42.066108937Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2002"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:2::1, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:42.066159618Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_70de9569-4ef0-4038-88e6-16ed7c5649a0","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:42.486802113Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3683"} {"msg":"configuring dendrite zone","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:42.492964241Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3739"} {"msg":"Ensuring address fd00:1122:3344:2::1 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:42.612638751Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2010"} {"msg":"Zone not using gateway (even though underlay is up)","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:42.612705012Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2034"} {"msg":"configuring wicketd","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:42.677960612Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3779","rack_subnet":"fd00:1122:3344::"} {"msg":"Determining physical location of our switch zone at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:42.72833107Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:360"} {"msg":"Failed to get switch ID from MGS (retrying in 33.366231ms)","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:11:42.760314474Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:379","error":"Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7e1ce7be-1ea8-4d01-99f3-855eb79cf49a\", \"content-length\": \"152\", \"date\": \"Mon, 26 Feb 2024 07:11:42 GMT\"}; value: Error { error_code: Some(\"DiscoveryNotYetComplete\"), message: \"discovery process not yet complete\", request_id: \"7e1ce7be-1ea8-4d01-99f3-855eb79cf49a\" }"} {"msg":"Setting up internal-dns service","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:42.791992042Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2187"} {"msg":"Failed to get switch ID from MGS (retrying in 74.83575ms)","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:11:42.795525264Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:379","error":"Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"cf4c631f-c3e4-4d48-b8c7-1f72443fddb7\", \"content-length\": \"152\", \"date\": \"Mon, 26 Feb 2024 07:11:42 GMT\"}; value: Error { error_code: Some(\"DiscoveryNotYetComplete\"), message: \"discovery process not yet complete\", request_id: \"cf4c631f-c3e4-4d48-b8c7-1f72443fddb7\" }"} {"msg":"Failed to get switch ID from MGS (retrying in 168.015704ms)","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:11:42.871462253Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:379","error":"Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"72ff8bd6-a730-4905-803a-4bb8341e84c6\", \"content-length\": \"152\", \"date\": \"Mon, 26 Feb 2024 07:11:42 GMT\"}; value: Error { error_code: Some(\"DiscoveryNotYetComplete\"), message: \"discovery process not yet complete\", request_id: \"72ff8bd6-a730-4905-803a-4bb8341e84c6\" }"} {"msg":"Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:43.041173114Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:408"} {"msg":"Building Port Configuration","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:43.073766898Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:567"} {"msg":"Checking dendrite uptime","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:43.07386003Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:609"} {"msg":"Dendrite online","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:43.074274398Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:612","uptime":"3"} {"msg":"Configuring default uplink on switch","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:43.074320688Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:430","config":"PortSettings {\n links: {\n \"0\": LinkSettings {\n addrs: [\n 10.151.1.174,\n ],\n params: LinkCreate {\n autoneg: false,\n fec: None,\n kr: false,\n lane: Some(\n LinkId(\n 0,\n ),\n ),\n speed: Speed40G,\n },\n },\n },\n v4_routes: {},\n v6_routes: {},\n}"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:43.136637867Z","hostname":"buskin","pid":1671,"sled_id":"5abda959-c9da-461c-83f2-8406a5d578e7","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"5abda959-c9da-461c-83f2-8406a5d578e7"} {"msg":"Received DisksChanged({})","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:43.151875505Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:43.26957886Z","hostname":"buskin","pid":1671,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:91","prefix":"Ipv6Prefix { addr: fd00:1122:3344:1::, len: 64 }"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:43.319446139Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_internal_dns_2a4f4adb-368e-4ad8-9ca4-08f3408cecf7"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:43.31950185Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_internal_dns_d7ff77b4-c6c4-4720-908c-7a0571f1e748"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:43.31950912Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_internal_dns_70de9569-4ef0-4038-88e6-16ed7c5649a0"} {"msg":"Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:43.319583781Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:198"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:43.319711773Z","hostname":"buskin","pid":1671,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:91","prefix":"Ipv6Prefix { addr: fd00:1122:3344:2::, len: 64 }"} {"msg":"Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:43.319963578Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:198"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:43.320271703Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"fdb0a792-4bdc-428b-80b6-fd6b52eef0d5","remote_addr":"[fd00:1122:3344:101::1]:62173","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":11136545,"response_code":"204"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:43.322029714Z","hostname":"buskin","pid":1671,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:417"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:43.367995175Z","hostname":"buskin","pid":1671,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:417"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:44.541316016Z","hostname":"buskin","pid":1671,"dns_config_addr":"[fd00:1122:3344:3::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:417"} {"msg":"Configured all DNS servers","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:44.586300648Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:453"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:44.586314298Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:254"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:44.61701829Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:271","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:44.617778443Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:296"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:44.617787053Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:188","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: d0d03c23-79c4-4c60-898e-739f188b5615, kind: Service { id: 3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc }, name: Name(\"ntp-3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 196, 59])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:44.645835808Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:44.646061332Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:40477"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:44.646340077Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:44.646351757Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:44.667976458Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:44.668063639Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:44.688212935Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:44.688283316Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:44.707342812Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:44.707401563Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:44.707627917Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:44.707761809Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:44.719450855Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:254"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:44.752358125Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:271","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:44.7532069Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:296"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:44.75322044Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:188","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:44.798077781Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte0\", ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 196, 59]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 }, vnic: \"vopte0\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:44.798129772Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1271","zone_type":"ntp"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.1.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:44.798876335Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:98"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.1.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:44.798885685Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:105"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.1.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:44.79913694Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:133"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:44.800397682Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Configuring new Omicron zone: oxz_ntp_3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:44.822504982Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_ntp_3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:44.84116015Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:45.88216923Z","hostname":"buskin","pid":1671,"zone":"oxz_ntp_3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_ntp_3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc\") failed: Property not found. retry in 68.251652ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:11:46.014098215Z","hostname":"buskin","pid":1671,"zone":"oxz_ntp_3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc\") failed: Property not found. retry in 100.462302ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:11:46.099929157Z","hostname":"buskin","pid":1671,"zone":"oxz_ntp_3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc\") failed: Property not found. retry in 216.952678ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:11:46.23282937Z","hostname":"buskin","pid":1671,"zone":"oxz_ntp_3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc\") failed: Property not found. retry in 498.073092ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:11:46.461162195Z","hostname":"buskin","pid":1671,"zone":"oxz_ntp_3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:46.521727152Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:59100"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:46.521885755Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"7ee3b9a0-3610-4dd5-92e7-9629a9109f4e","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:59100","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":72,"response_code":"200"} {"msg":"wait for service Some(\"oxz_ntp_3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc\") failed: Property not found. retry in 720.158058ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:11:46.969321611Z","hostname":"buskin","pid":1671,"zone":"oxz_ntp_3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc\") failed: Property not found. retry in 621.681122ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:11:47.700033841Z","hostname":"buskin","pid":1671,"zone":"oxz_ntp_3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc\") failed: Property not found. retry in 1.390244207s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:11:48.331086974Z","hostname":"buskin","pid":1671,"zone":"oxz_ntp_3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc\") failed: Property not found. retry in 1.22329268s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:11:49.73160428Z","hostname":"buskin","pid":1671,"zone":"oxz_ntp_3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring address fd00:1122:3344:101::1b exists","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:50.995811112Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2002"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::1b, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:50.995863173Z","hostname":"buskin","pid":1671,"zone":"oxz_ntp_3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"Ensuring address fd00:1122:3344:101::1b exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:51.546410088Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2010"} {"msg":"Zone using sled underlay as gateway","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:51.546461078Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2029"} {"msg":"Set up NTP service boundary=true, Servers=[\"0.pool.ntp.org\"]","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:51.7456597Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2270"} {"msg":"Ensuring address for OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:51.772193017Z","hostname":"buskin","pid":1671,"zone":"oxz_ntp_3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:653"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:56.497046657Z","hostname":"buskin","pid":1671,"sled_id":"5abda959-c9da-461c-83f2-8406a5d578e7","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"5abda959-c9da-461c-83f2-8406a5d578e7"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:56.553614514Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_ntp_3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc"} {"msg":"Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:56.553766327Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:198"} {"msg":"Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:56.554131613Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:198"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:56.554403638Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"1aa6984a-02a6-49c0-9a0a-db8df52fd3bf","remote_addr":"[fd00:1122:3344:101::1]:40477","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":11908291,"response_code":"204"} {"msg":"Waiting for rack time synchronization","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:56.557113056Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:494"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:56.58741042Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:56.587585553Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:65042"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:56.597134501Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3393"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:56.597260723Z","hostname":"buskin","pid":1671,"uri":"/timesync","method":"GET","req_id":"befad08f-b487-4ba7-b86c-26714c434b0a","remote_addr":"[fd00:1122:3344:101::1]:65042","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":9413,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:11:56.600205526Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:56.729433273Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:56.729566715Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:52015"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:56.738410771Z","hostname":"buskin","pid":1671,"uri":"/timesync","method":"GET","req_id":"9e305b9c-2e17-4008-a689-18d9c8ce3690","remote_addr":"[fd00:1122:3344:101::1]:52015","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":8710,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1708931515.5480435, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:56.740257324Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:11:56.740265054Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:56.937184725Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:56.937318108Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:32811"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:56.945765026Z","hostname":"buskin","pid":1671,"uri":"/timesync","method":"GET","req_id":"a47e8a44-10a9-4b63-8f36-43b555eef8b3","remote_addr":"[fd00:1122:3344:101::1]:32811","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":8326,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1708931515.5480435, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:56.947641039Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:11:56.94764797Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:57.118719795Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:57.118835787Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:54894"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:57.126468211Z","hostname":"buskin","pid":1671,"uri":"/timesync","method":"GET","req_id":"50cbfdae-a1d6-4c79-8011-6722d0b6e0dd","remote_addr":"[fd00:1122:3344:101::1]:54894","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":7489,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1708931515.5480435, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:57.128210222Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:11:57.128218602Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:57.276962664Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:57.277085146Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:41249"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:57.284769842Z","hostname":"buskin","pid":1671,"uri":"/timesync","method":"GET","req_id":"ea85fb1e-7e41-48d0-aef1-322654e09699","remote_addr":"[fd00:1122:3344:101::1]:41249","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":7559,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1708931515.5480435, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:57.287663513Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:11:57.287669813Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:57.428995654Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:57.429102126Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:63462"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:57.437332581Z","hostname":"buskin","pid":1671,"uri":"/timesync","method":"GET","req_id":"cf7f39fa-ebac-44c6-a77d-e61f7703c8ed","remote_addr":"[fd00:1122:3344:101::1]:63462","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":8117,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1708931515.5480435, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:57.438931869Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:11:57.438939829Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:57.688341115Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:57.688474688Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:47495"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:57.696987327Z","hostname":"buskin","pid":1671,"uri":"/timesync","method":"GET","req_id":"3467847f-4a12-412f-9fec-c048a972a56e","remote_addr":"[fd00:1122:3344:101::1]:47495","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":8369,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1708931515.5480435, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:57.698964122Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:11:57.698971122Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:58.146925558Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:58.14702208Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:46099"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:58.155120742Z","hostname":"buskin","pid":1671,"uri":"/timesync","method":"GET","req_id":"9efb57ce-3d3a-40a9-a0ce-666de1af0b3c","remote_addr":"[fd00:1122:3344:101::1]:46099","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":7963,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1708931515.5480435, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:58.156633579Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:11:58.156638869Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:58.397376572Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:58.397502835Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:55608"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:58.40628248Z","hostname":"buskin","pid":1671,"uri":"/timesync","method":"GET","req_id":"b18bd203-ed9c-4937-8058-52b934ebd01c","remote_addr":"[fd00:1122:3344:101::1]:55608","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":8688,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1708931515.5480435, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:58.408222884Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:11:58.408230494Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:58.712526558Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:58.712638759Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:34115"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:58.720723012Z","hostname":"buskin","pid":1671,"uri":"/timesync","method":"GET","req_id":"79f90bf2-77db-4fbb-843c-7056f8160937","remote_addr":"[fd00:1122:3344:101::1]:34115","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":7939,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1708931515.5480435, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:58.722668006Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:11:58.722673636Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:59.400814089Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:59.400934821Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:57257"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:59.409179217Z","hostname":"buskin","pid":1671,"uri":"/timesync","method":"GET","req_id":"8efefcc1-3c4c-495a-ae3e-b2940748feb9","remote_addr":"[fd00:1122:3344:101::1]:57257","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":8089,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1708931515.5480435, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:59.410998959Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:11:59.411018219Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:59.839471541Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:59.839582853Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:63073"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:59.848239876Z","hostname":"buskin","pid":1671,"uri":"/timesync","method":"GET","req_id":"17b7e2b4-4deb-43b8-a8ea-91e381301ee4","remote_addr":"[fd00:1122:3344:101::1]:63073","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":8503,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1708931515.5480435, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:11:59.849961126Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:11:59.849966916Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:00.485127201Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:00.485232283Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:50520"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:00.493366216Z","hostname":"buskin","pid":1671,"uri":"/timesync","method":"GET","req_id":"d6f3dc62-44fd-4681-a6a8-726a18a484df","remote_addr":"[fd00:1122:3344:101::1]:50520","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":7982,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1708931515.5480435, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:00.495172388Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:00.495178538Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:01.058012499Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:01.058190382Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:44437"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:01.067540456Z","hostname":"buskin","pid":1671,"uri":"/timesync","method":"GET","req_id":"2af9c180-c4de-45ff-ac5c-9c3704ea6cbc","remote_addr":"[fd00:1122:3344:101::1]:44437","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":9308,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1708931515.5480435, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:01.0711713Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:01.071178061Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:02.634210221Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:02.634328023Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:61161"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:02.642538058Z","hostname":"buskin","pid":1671,"uri":"/timesync","method":"GET","req_id":"cb1cfc4a-d933-4102-8f2d-a69979931017","remote_addr":"[fd00:1122:3344:101::1]:61161","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":8057,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1708931515.5480435, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:02.64437571Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:02.64438354Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:04.407430251Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:04.407557413Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:36776"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:04.411113176Z","hostname":"buskin","pid":1671,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:776"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:04.428980581Z","hostname":"buskin","pid":1671,"uri":"/timesync","method":"GET","req_id":"f38e405c-1de4-4646-873a-57bf826e39e5","remote_addr":"[fd00:1122:3344:101::1]:36776","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":21261,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: true, ref_id: 265050079, ip_addr: 15.204.87.223, stratum: 3, ref_time: 1708931522.596233, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:04.430942446Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Finished setting up Internal DNS and NTP","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:04.430952276Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1039"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: d0d03c23-79c4-4c60-898e-739f188b5615, kind: Service { id: 3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc }, name: Name(\"ntp-3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 196, 59])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 4b099cd4-61db-4587-84b8-cf9015577106, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 4c8e127e-7304-49fa-9de2-79bbde180d94, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:04.459588391Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:04.459856595Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:38474"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:04.460285333Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:04.460303553Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:04.481933624Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:04.481998215Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:04.501614581Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:04.501662602Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:04.51857806Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:04.518624241Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:04.569270884Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:04.569327235Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b099cd4-61db-4587-84b8-cf9015577106, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:04.619727863Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4b099cd4-61db-4587-84b8-cf9015577106, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:04.619783774Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:04.671084488Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:04.671138559Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4c8e127e-7304-49fa-9de2-79bbde180d94, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:04.725611149Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4c8e127e-7304-49fa-9de2-79bbde180d94, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:04.72566153Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:04.777133288Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:04.777191619Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:04.777387982Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:04.777514314Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:04.859609791Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:04.859699243Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:04.859708683Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:04.859717823Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:04.859749254Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_4b099cd4-61db-4587-84b8-cf9015577106","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:04.952648961Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_89b5827c-8fa9-4662-9e35-ddf672d463bc","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:04.957001408Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_4c8e127e-7304-49fa-9de2-79bbde180d94","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:04.961323604Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_11ee5b01-c0d0-44c9-b381-52b9b32cde90","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:04.965701091Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_dab2ea04-af43-4f0f-8d35-5d21dc9d8253","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:04.97014066Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_cockroachdb_4b099cd4-61db-4587-84b8-cf9015577106","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:04.974905144Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_89b5827c-8fa9-4662-9e35-ddf672d463bc","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:04.979394843Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_4c8e127e-7304-49fa-9de2-79bbde180d94","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:04.983500585Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_11ee5b01-c0d0-44c9-b381-52b9b32cde90","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:04.987902503Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_dab2ea04-af43-4f0f-8d35-5d21dc9d8253","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:04.992361741Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_cockroachdb_4b099cd4-61db-4587-84b8-cf9015577106:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:08.391057294Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:08.391247258Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_4b099cd4-61db-4587-84b8-cf9015577106","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:08.394440664Z","hostname":"buskin","pid":1671,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:776"} {"msg":"Profile for oxz_cockroachdb_11ee5b01-c0d0-44c9-b381-52b9b32cde90:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:08.415106369Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:08.415297152Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_11ee5b01-c0d0-44c9-b381-52b9b32cde90","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_cockroachdb_4b099cd4-61db-4587-84b8-cf9015577106\") failed: Property not found. retry in 53.873782ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:08.53770625Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_4b099cd4-61db-4587-84b8-cf9015577106","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_11ee5b01-c0d0-44c9-b381-52b9b32cde90\") failed: Property not found. retry in 31.401106ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:08.557359226Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_11ee5b01-c0d0-44c9-b381-52b9b32cde90","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_cockroachdb_4c8e127e-7304-49fa-9de2-79bbde180d94:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:08.560295438Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:08.5604383Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_4c8e127e-7304-49fa-9de2-79bbde180d94","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_cockroachdb_89b5827c-8fa9-4662-9e35-ddf672d463bc:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:08.568296599Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:08.568463652Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_89b5827c-8fa9-4662-9e35-ddf672d463bc","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_cockroachdb_11ee5b01-c0d0-44c9-b381-52b9b32cde90\") failed: Property not found. retry in 58.277195ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:08.598639813Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_11ee5b01-c0d0-44c9-b381-52b9b32cde90","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4b099cd4-61db-4587-84b8-cf9015577106\") failed: Property not found. retry in 119.933006ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:08.619195226Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_4b099cd4-61db-4587-84b8-cf9015577106","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_11ee5b01-c0d0-44c9-b381-52b9b32cde90\") failed: Property not found. retry in 204.360988ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:08.771377438Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_11ee5b01-c0d0-44c9-b381-52b9b32cde90","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4b099cd4-61db-4587-84b8-cf9015577106\") failed: Property not found. retry in 243.597788ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:08.817703515Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_4b099cd4-61db-4587-84b8-cf9015577106","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_cockroachdb_dab2ea04-af43-4f0f-8d35-5d21dc9d8253:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:08.828278241Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:08.881882476Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_dab2ea04-af43-4f0f-8d35-5d21dc9d8253","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_cockroachdb_4c8e127e-7304-49fa-9de2-79bbde180d94\") failed: Property not found. retry in 51.244212ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:09.199032056Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_4c8e127e-7304-49fa-9de2-79bbde180d94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4b099cd4-61db-4587-84b8-cf9015577106\") failed: Property not found. retry in 355.64342ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:09.21057673Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_4b099cd4-61db-4587-84b8-cf9015577106","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_89b5827c-8fa9-4662-9e35-ddf672d463bc\") failed: Property not found. retry in 56.043035ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:09.258311131Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_89b5827c-8fa9-4662-9e35-ddf672d463bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_11ee5b01-c0d0-44c9-b381-52b9b32cde90\") failed: Property not found. retry in 440.288468ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:09.292615965Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_11ee5b01-c0d0-44c9-b381-52b9b32cde90","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4c8e127e-7304-49fa-9de2-79bbde180d94\") failed: Property not found. retry in 72.484387ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:09.325586247Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_4c8e127e-7304-49fa-9de2-79bbde180d94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_89b5827c-8fa9-4662-9e35-ddf672d463bc\") failed: Property not found. retry in 120.645437ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:09.350084449Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_89b5827c-8fa9-4662-9e35-ddf672d463bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4c8e127e-7304-49fa-9de2-79bbde180d94\") failed: Property not found. retry in 157.366681ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:09.419284098Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_4c8e127e-7304-49fa-9de2-79bbde180d94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_dab2ea04-af43-4f0f-8d35-5d21dc9d8253\") failed: Property not found. retry in 48.841311ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:09.471366416Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_dab2ea04-af43-4f0f-8d35-5d21dc9d8253","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_89b5827c-8fa9-4662-9e35-ddf672d463bc\") failed: Property not found. retry in 257.279638ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:09.501284503Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_89b5827c-8fa9-4662-9e35-ddf672d463bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_dab2ea04-af43-4f0f-8d35-5d21dc9d8253\") failed: Property not found. retry in 102.95346ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:09.542012691Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_dab2ea04-af43-4f0f-8d35-5d21dc9d8253","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4b099cd4-61db-4587-84b8-cf9015577106\") failed: Property not found. retry in 1.001708692s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:09.649470306Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_4b099cd4-61db-4587-84b8-cf9015577106","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4c8e127e-7304-49fa-9de2-79bbde180d94\") failed: Property not found. retry in 477.654878ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:09.725489805Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_4c8e127e-7304-49fa-9de2-79bbde180d94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_dab2ea04-af43-4f0f-8d35-5d21dc9d8253\") failed: Property not found. retry in 278.940513ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:09.786791036Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_dab2ea04-af43-4f0f-8d35-5d21dc9d8253","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_89b5827c-8fa9-4662-9e35-ddf672d463bc\") failed: Property not found. retry in 563.044213ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:09.840392651Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_89b5827c-8fa9-4662-9e35-ddf672d463bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_11ee5b01-c0d0-44c9-b381-52b9b32cde90\") failed: Property not found. retry in 540.91052ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:10.1076065Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_11ee5b01-c0d0-44c9-b381-52b9b32cde90","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_dab2ea04-af43-4f0f-8d35-5d21dc9d8253\") failed: Property not found. retry in 497.342915ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:10.247272062Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_dab2ea04-af43-4f0f-8d35-5d21dc9d8253","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4c8e127e-7304-49fa-9de2-79bbde180d94\") failed: Property not found. retry in 880.564539ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:10.447261497Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_4c8e127e-7304-49fa-9de2-79bbde180d94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_89b5827c-8fa9-4662-9e35-ddf672d463bc\") failed: Property not found. retry in 545.01839ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:10.789333797Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_89b5827c-8fa9-4662-9e35-ddf672d463bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4b099cd4-61db-4587-84b8-cf9015577106\") failed: Property not found. retry in 1.370344688s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:10.877196165Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_4b099cd4-61db-4587-84b8-cf9015577106","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_11ee5b01-c0d0-44c9-b381-52b9b32cde90\") failed: Property not found. retry in 747.589419ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:10.897157557Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_11ee5b01-c0d0-44c9-b381-52b9b32cde90","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_dab2ea04-af43-4f0f-8d35-5d21dc9d8253\") failed: Property not found. retry in 796.983716ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:10.912910585Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_dab2ea04-af43-4f0f-8d35-5d21dc9d8253","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4c8e127e-7304-49fa-9de2-79bbde180d94\") failed: Property not found. retry in 1.238062544s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:11.285050444Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_4c8e127e-7304-49fa-9de2-79bbde180d94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_89b5827c-8fa9-4662-9e35-ddf672d463bc\") failed: Property not found. retry in 1.261615638s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:11.344852628Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_89b5827c-8fa9-4662-9e35-ddf672d463bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_11ee5b01-c0d0-44c9-b381-52b9b32cde90\") failed: Property not found. retry in 1.178817455s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:11.656000202Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_11ee5b01-c0d0-44c9-b381-52b9b32cde90","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_dab2ea04-af43-4f0f-8d35-5d21dc9d8253\") failed: Property not found. retry in 564.70496ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:11.720742353Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_dab2ea04-af43-4f0f-8d35-5d21dc9d8253","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4b099cd4-61db-4587-84b8-cf9015577106\") failed: Property not found. retry in 1.104179443s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:12.258297388Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_4b099cd4-61db-4587-84b8-cf9015577106","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_dab2ea04-af43-4f0f-8d35-5d21dc9d8253\") failed: Property not found. retry in 1.470366506s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:12.29469251Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_dab2ea04-af43-4f0f-8d35-5d21dc9d8253","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4c8e127e-7304-49fa-9de2-79bbde180d94\") failed: Property not found. retry in 1.292922134s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:12.533047961Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_4c8e127e-7304-49fa-9de2-79bbde180d94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_89b5827c-8fa9-4662-9e35-ddf672d463bc\") failed: Property not found. retry in 643.219645ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:12.615684507Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_89b5827c-8fa9-4662-9e35-ddf672d463bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_11ee5b01-c0d0-44c9-b381-52b9b32cde90\") failed: Property not found. retry in 1.332153693s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:12.844828946Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_11ee5b01-c0d0-44c9-b381-52b9b32cde90","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_89b5827c-8fa9-4662-9e35-ddf672d463bc\") failed: Property not found. retry in 1.090915269s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:13.268649357Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_89b5827c-8fa9-4662-9e35-ddf672d463bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4b099cd4-61db-4587-84b8-cf9015577106\") failed: Property not found. retry in 809.611594ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:13.372063819Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_4b099cd4-61db-4587-84b8-cf9015577106","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_dab2ea04-af43-4f0f-8d35-5d21dc9d8253\") failed: Property not found. retry in 1.119507404s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:13.773647127Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_dab2ea04-af43-4f0f-8d35-5d21dc9d8253","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4c8e127e-7304-49fa-9de2-79bbde180d94\") failed: Property not found. retry in 1.10493222s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:13.835529088Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_4c8e127e-7304-49fa-9de2-79bbde180d94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_11ee5b01-c0d0-44c9-b381-52b9b32cde90\") failed: Property not found. retry in 1.47636692s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:14.189159971Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_11ee5b01-c0d0-44c9-b381-52b9b32cde90","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4b099cd4-61db-4587-84b8-cf9015577106\") failed: Property not found. retry in 671.58717ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:14.201403897Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_4b099cd4-61db-4587-84b8-cf9015577106","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_89b5827c-8fa9-4662-9e35-ddf672d463bc\") failed: Property not found. retry in 976.600466ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:14.371380413Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_89b5827c-8fa9-4662-9e35-ddf672d463bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4b099cd4-61db-4587-84b8-cf9015577106\") failed: Property not found. retry in 1.408491305s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:14.884434386Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_4b099cd4-61db-4587-84b8-cf9015577106","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_dab2ea04-af43-4f0f-8d35-5d21dc9d8253\") failed: Property not found. retry in 1.271370841s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:14.902688578Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_dab2ea04-af43-4f0f-8d35-5d21dc9d8253","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4c8e127e-7304-49fa-9de2-79bbde180d94\") failed: Property not found. retry in 1.088809091s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:14.948115379Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_4c8e127e-7304-49fa-9de2-79bbde180d94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_89b5827c-8fa9-4662-9e35-ddf672d463bc\") failed: Property not found. retry in 1.065289164s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:15.356349594Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_89b5827c-8fa9-4662-9e35-ddf672d463bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_11ee5b01-c0d0-44c9-b381-52b9b32cde90\") failed: Property not found. retry in 512.545592ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:15.674159025Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_11ee5b01-c0d0-44c9-b381-52b9b32cde90","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4c8e127e-7304-49fa-9de2-79bbde180d94\") failed: Property not found. retry in 1.442703972s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:16.045355148Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_4c8e127e-7304-49fa-9de2-79bbde180d94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_dab2ea04-af43-4f0f-8d35-5d21dc9d8253\") failed: Property not found. retry in 1.216509746s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:16.181953456Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_dab2ea04-af43-4f0f-8d35-5d21dc9d8253","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_11ee5b01-c0d0-44c9-b381-52b9b32cde90\") failed: Property not found. retry in 1.362549643s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:16.195333682Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_11ee5b01-c0d0-44c9-b381-52b9b32cde90","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4b099cd4-61db-4587-84b8-cf9015577106\") failed: Property not found. retry in 675.4202ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:16.300444884Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_4b099cd4-61db-4587-84b8-cf9015577106","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_89b5827c-8fa9-4662-9e35-ddf672d463bc\") failed: Property not found. retry in 694.295069ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:16.431679467Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_89b5827c-8fa9-4662-9e35-ddf672d463bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4b099cd4-61db-4587-84b8-cf9015577106\") failed: Property not found. retry in 671.802442ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:16.985729183Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_4b099cd4-61db-4587-84b8-cf9015577106","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_89b5827c-8fa9-4662-9e35-ddf672d463bc\") failed: Property not found. retry in 1.278890339s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:17.136026982Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_89b5827c-8fa9-4662-9e35-ddf672d463bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_dab2ea04-af43-4f0f-8d35-5d21dc9d8253\") failed: Property not found. retry in 1.218028267s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:17.408639217Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_dab2ea04-af43-4f0f-8d35-5d21dc9d8253","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4c8e127e-7304-49fa-9de2-79bbde180d94\") failed: Property not found. retry in 812.567161ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:17.498636583Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_4c8e127e-7304-49fa-9de2-79bbde180d94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:17.555294362Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:45628"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:17.555424974Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"fe9a0af2-2854-4a80-8a05-1b34c1e4e6d6","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:45628","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":77,"response_code":"200"} {"msg":"wait for service Some(\"oxz_cockroachdb_11ee5b01-c0d0-44c9-b381-52b9b32cde90\") failed: Property not found. retry in 1.475625028s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:17.569226718Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_11ee5b01-c0d0-44c9-b381-52b9b32cde90","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4b099cd4-61db-4587-84b8-cf9015577106\") failed: Property not found. retry in 626.692727ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:17.667885977Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_4b099cd4-61db-4587-84b8-cf9015577106","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4b099cd4-61db-4587-84b8-cf9015577106\") failed: Property not found. retry in 619.753686ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:18.304786252Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_4b099cd4-61db-4587-84b8-cf9015577106","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4c8e127e-7304-49fa-9de2-79bbde180d94\") failed: Property not found. retry in 589.487533ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:18.320343947Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_4c8e127e-7304-49fa-9de2-79bbde180d94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4c8e127e-7304-49fa-9de2-79bbde180d94\") failed: Property not found. retry in 582.896249ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:18.918026001Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_4c8e127e-7304-49fa-9de2-79bbde180d94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4b099cd4-61db-4587-84b8-cf9015577106\") failed: Property not found. retry in 1.304141318s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:18.932842013Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_4b099cd4-61db-4587-84b8-cf9015577106","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4c8e127e-7304-49fa-9de2-79bbde180d94\") failed: Property not found. retry in 681.092752ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:19.511735806Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_4c8e127e-7304-49fa-9de2-79bbde180d94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4c8e127e-7304-49fa-9de2-79bbde180d94\") failed: Property not found. retry in 1.49188296s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:20.457917623Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_4c8e127e-7304-49fa-9de2-79bbde180d94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4b099cd4-61db-4587-84b8-cf9015577106\") failed: Property not found. retry in 602.881933ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:20.471425292Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_4b099cd4-61db-4587-84b8-cf9015577106","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4b099cd4-61db-4587-84b8-cf9015577106\") failed: Property not found. retry in 1.363827269s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:21.086000694Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_4b099cd4-61db-4587-84b8-cf9015577106","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4c8e127e-7304-49fa-9de2-79bbde180d94\") failed: Property not found. retry in 611.364221ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:21.960694531Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_4c8e127e-7304-49fa-9de2-79bbde180d94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4b099cd4-61db-4587-84b8-cf9015577106\") failed: Property not found. retry in 1.187984727s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:22.461167152Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_4b099cd4-61db-4587-84b8-cf9015577106","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4c8e127e-7304-49fa-9de2-79bbde180d94\") failed: Property not found. retry in 654.44213ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:22.724637806Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_4c8e127e-7304-49fa-9de2-79bbde180d94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4c8e127e-7304-49fa-9de2-79bbde180d94\") failed: Property not found. retry in 755.336594ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:23.388740372Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_4c8e127e-7304-49fa-9de2-79bbde180d94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4b099cd4-61db-4587-84b8-cf9015577106\") failed: Property not found. retry in 545.443766ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:23.65869494Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_4b099cd4-61db-4587-84b8-cf9015577106","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4c8e127e-7304-49fa-9de2-79bbde180d94\") failed: Property not found. retry in 1.06766454s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:24.153613873Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_4c8e127e-7304-49fa-9de2-79bbde180d94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4b099cd4-61db-4587-84b8-cf9015577106\") failed: Property not found. retry in 1.016570789s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:24.212664834Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_4b099cd4-61db-4587-84b8-cf9015577106","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4c8e127e-7304-49fa-9de2-79bbde180d94\") failed: Property not found. retry in 633.277918ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:25.231571883Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_4c8e127e-7304-49fa-9de2-79bbde180d94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4b099cd4-61db-4587-84b8-cf9015577106\") failed: Property not found. retry in 624.605155ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:25.239645725Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_4b099cd4-61db-4587-84b8-cf9015577106","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4b099cd4-61db-4587-84b8-cf9015577106\") failed: Property not found. retry in 1.322233922s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:25.874550006Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_4b099cd4-61db-4587-84b8-cf9015577106","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4c8e127e-7304-49fa-9de2-79bbde180d94\") failed: Property not found. retry in 584.084539ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:25.882519917Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_4c8e127e-7304-49fa-9de2-79bbde180d94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4c8e127e-7304-49fa-9de2-79bbde180d94\") failed: Property not found. retry in 1.132747742s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:26.47558385Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_4c8e127e-7304-49fa-9de2-79bbde180d94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4b099cd4-61db-4587-84b8-cf9015577106\") failed: Property not found. retry in 1.011360803s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:27.205443394Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_4b099cd4-61db-4587-84b8-cf9015577106","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4c8e127e-7304-49fa-9de2-79bbde180d94\") failed: Property not found. retry in 1.351908253s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:27.619315179Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_4c8e127e-7304-49fa-9de2-79bbde180d94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:28.991962463Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_cockroachdb_4b099cd4-61db-4587-84b8-cf9015577106"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:28.992056305Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_cockroachdb_89b5827c-8fa9-4662-9e35-ddf672d463bc"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:28.992066475Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_cockroachdb_4c8e127e-7304-49fa-9de2-79bbde180d94"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:28.992075655Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_cockroachdb_11ee5b01-c0d0-44c9-b381-52b9b32cde90"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:28.992082085Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_cockroachdb_dab2ea04-af43-4f0f-8d35-5d21dc9d8253"} {"msg":"Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:28.992090445Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:198"} {"msg":"Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:28.992491672Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:198"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:28.992792488Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"62ca7686-34ae-4f2b-a215-d78f17d67d0f","remote_addr":"[fd00:1122:3344:101::1]:38474","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":24555546,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:29.027179554Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:52256"} {"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::4]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:29.06040614Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3227"} {"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:29.393322668Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3245"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:39.18265911Z","hostname":"buskin","pid":1671,"uri":"/cockroachdb","method":"POST","req_id":"797f6f07-87ee-4d0d-b05e-f09b5777e2f7","remote_addr":"[fd00:1122:3344:101::1]:52256","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize CockroachDb: Error running command in zone 'oxz_cockroachdb_11ee5b01-c0d0-44c9-b381-52b9b32cde90': Command [/opt/oxide/cockroachdb/bin/cockroach sql --insecure --host [fd00:1122:3344:101::4]:32221 --file /opt/oxide/cockroachdb/sql/dbwipe.sql] executed and failed with status: exit status: 1 stdout: BEGIN\nCREATE DATABASE\nCREATE ROLE\nALTER DEFAULT PRIVILEGES\n stderr: ERROR: restart transaction: TransactionRetryWithProtoRefreshError: TransactionAbortedError(ABORT_REASON_CLIENT_REJECT): \"sql txn\" meta={id=71172da5 key=/NamespaceTable/30/1/104/0/\"public\"/4/1 pri=0.01429971 epo=0 ts=1708931554.902276031,2 min=1708931550.192265559,0 seq=31} lock=true stat=PENDING rts=1708931550.192265559,0 wto=false gul=1708931550.692265559,0\nSQLSTATE: 40001\nHINT: See: https://www.cockroachlabs.com/docs/v22.1/transaction-retry-error-reference.html#abort_reason_client_reject\nERROR: restart transaction: TransactionRetryWithProtoRefreshError: TransactionAbortedError(ABORT_REASON_CLIENT_REJECT): \"sql txn\" meta={id=71172da5 key=/NamespaceTable/30/1/104/0/\"public\"/4/1 pri=0.01429971 epo=0 ts=1708931554.902276031,2 min=1708931550.192265559,0 seq=31} lock=true stat=PENDING rts=1708931550.192265559,0 wto=false gul=1708931550.692265559,0\nSQLSTATE: 40001\nHINT: See: https://www.cockroachlabs.com/docs/v22.1/transaction-retry-error-reference.html#abort_reason_client_reject\nFailed running \"sql\"\n","latency_us":10377806,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:39.185073053Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:755","retry_after":"54.317611ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"797f6f07-87ee-4d0d-b05e-f09b5777e2f7\", \"content-length\": \"124\", \"date\": \"Mon, 26 Feb 2024 07:12:39 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"797f6f07-87ee-4d0d-b05e-f09b5777e2f7\" }"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:39.244441778Z","hostname":"buskin","pid":1671,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:776"} {"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::4]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:39.283068335Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3227"} {"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:39.908516929Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3245"} {"msg":"Formatting CRDB - Completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:47.534119461Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3268"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:47.534214173Z","hostname":"buskin","pid":1671,"uri":"/cockroachdb","method":"POST","req_id":"1084679f-4cd6-46d8-ad2d-497c4f01d652","remote_addr":"[fd00:1122:3344:101::1]:52256","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":8316788,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:47.534593629Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:50528"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:47.534820484Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"c4614b9c-b178-4ff0-b56e-9254a3d17228","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:50528","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":128,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: d0d03c23-79c4-4c60-898e-739f188b5615, kind: Service { id: 3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc }, name: Name(\"ntp-3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 196, 59])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 4b099cd4-61db-4587-84b8-cf9015577106, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 4c8e127e-7304-49fa-9de2-79bbde180d94, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: d60f3c38-1add-47dd-baca-a6103e9614ae, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 864c4b26-c35e-42dc-a5b0-3e041205ccbc, kind: Service { id: d60f3c38-1add-47dd-baca-a6103e9614ae }, name: Name(\"external-dns-d60f3c38-1add-47dd-baca-a6103e9614ae\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 7])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: c9ac5e72-089e-4e66-8c75-d1c69a21aee1, kind: Service { id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95 }, name: Name(\"external-dns-740e2473-798e-4bf1-9bf5-7a2c9ce38b95\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 248])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 60944a67-2d69-42d2-8ce7-1215cdc06eab, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 850653ea-9272-4e6c-b3ae-b169328ac41c, kind: Service { id: 60944a67-2d69-42d2-8ce7-1215cdc06eab }, name: Name(\"nexus-60944a67-2d69-42d2-8ce7-1215cdc06eab\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 217, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 3c85fee5-2c8b-46c2-a723-6c621ecfb3be, kind: Service { id: 3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7 }, name: Name(\"nexus-3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 215])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e72f50ee-9b20-4af9-becc-46f37972a3b0, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 03f00562-72e1-4d42-8124-11bb07005618, kind: Service { id: e72f50ee-9b20-4af9-becc-46f37972a3b0 }, name: Name(\"nexus-e72f50ee-9b20-4af9-becc-46f37972a3b0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 176, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7fcbadc5-146c-4a16-bbe8-9fe02e4fe911, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: fbaa903b-4740-4365-b35e-68123cc1d3a2, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1693d50c-fcf2-4249-86ae-6e12a08ac868, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 065d24ba-ce47-47d1-9175-26eabb479878, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: bc1764f3-5054-4735-919d-fb5def7e7e42, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a493457b-acb1-4029-bcc4-1d8290fa8759, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 1daa94e0-684d-4a00-9332-60a34c24e94a, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: fdc9e743-1329-4663-9bc9-0b2a07234456, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:47.573310144Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:47.574751769Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:42546"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:47.575560393Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:47.575575984Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:47.599406525Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:47.599451356Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:47.620552809Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:47.620594089Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:47.641518809Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:47.64155955Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:47.660410063Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:47.660447934Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b099cd4-61db-4587-84b8-cf9015577106, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:47.678833129Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4b099cd4-61db-4587-84b8-cf9015577106, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:47.678876159Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:47.696147885Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:47.696188316Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4c8e127e-7304-49fa-9de2-79bbde180d94, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:47.714315506Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4c8e127e-7304-49fa-9de2-79bbde180d94, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:47.714356546Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d60f3c38-1add-47dd-baca-a6103e9614ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:47.731718463Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d60f3c38-1add-47dd-baca-a6103e9614ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:47.731759634Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:47.873692762Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:47.873735413Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:48.000073576Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:48.000118857Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bc1764f3-5054-4735-919d-fb5def7e7e42, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:48.119647539Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bc1764f3-5054-4735-919d-fb5def7e7e42, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:48.1197027Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a493457b-acb1-4029-bcc4-1d8290fa8759, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:48.230868825Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a493457b-acb1-4029-bcc4-1d8290fa8759, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:48.230917206Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:48.38846013Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:48.388511361Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:48.539437509Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:48.5394835Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1daa94e0-684d-4a00-9332-60a34c24e94a, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:48.720728063Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1daa94e0-684d-4a00-9332-60a34c24e94a, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:48.720784744Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:48.879643141Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:48.879688272Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:49.020289457Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:49.020335268Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fdc9e743-1329-4663-9bc9-0b2a07234456, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:49.161095226Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fdc9e743-1329-4663-9bc9-0b2a07234456, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:49.161147647Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:49.289377003Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:49.289421454Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:49.428904656Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:49.428950269Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:49.429201614Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:49.429350287Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:49.564916323Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:49.564970333Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:49.564978594Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:49.564986084Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:49.564991784Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:49.564997144Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:49.565018704Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:49.565024694Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:49.565031495Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:49.565035885Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:49.565041575Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:49.565045885Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:49.565051375Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:49.565055555Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:49.565062465Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:49.565067795Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:49.565073085Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:49.565078535Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:49.565083806Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:49.565208168Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"8852bff7-fbb6-4cd8-a65c-a14dc90ce391","remote_addr":"[fd00:1122:3344:101::1]:42546","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\", TimeNotSynchronized), (\"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868\", TimeNotSynchronized), (\"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae\", TimeNotSynchronized), (\"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42\", TimeNotSynchronized), (\"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47\", TimeNotSynchronized), (\"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7\", TimeNotSynchronized), (\"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0\", TimeNotSynchronized), (\"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846\", TimeNotSynchronized), (\"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878\", TimeNotSynchronized), (\"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a\", TimeNotSynchronized), (\"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab\", TimeNotSynchronized), (\"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c\", TimeNotSynchronized), (\"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95\", TimeNotSynchronized), (\"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911\", TimeNotSynchronized), (\"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff\", TimeNotSynchronized), (\"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759\", TimeNotSynchronized), (\"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2\", TimeNotSynchronized), (\"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456\", TimeNotSynchronized), (\"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af\", TimeNotSynchronized)]","latency_us":1995742,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:49.565488263Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"149.237187ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8852bff7-fbb6-4cd8-a65c-a14dc90ce391\", \"content-length\": \"124\", \"date\": \"Mon, 26 Feb 2024 07:12:49 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"8852bff7-fbb6-4cd8-a65c-a14dc90ce391\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: d0d03c23-79c4-4c60-898e-739f188b5615, kind: Service { id: 3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc }, name: Name(\"ntp-3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 196, 59])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 4b099cd4-61db-4587-84b8-cf9015577106, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 4c8e127e-7304-49fa-9de2-79bbde180d94, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: d60f3c38-1add-47dd-baca-a6103e9614ae, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 864c4b26-c35e-42dc-a5b0-3e041205ccbc, kind: Service { id: d60f3c38-1add-47dd-baca-a6103e9614ae }, name: Name(\"external-dns-d60f3c38-1add-47dd-baca-a6103e9614ae\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 7])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: c9ac5e72-089e-4e66-8c75-d1c69a21aee1, kind: Service { id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95 }, name: Name(\"external-dns-740e2473-798e-4bf1-9bf5-7a2c9ce38b95\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 248])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 60944a67-2d69-42d2-8ce7-1215cdc06eab, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 850653ea-9272-4e6c-b3ae-b169328ac41c, kind: Service { id: 60944a67-2d69-42d2-8ce7-1215cdc06eab }, name: Name(\"nexus-60944a67-2d69-42d2-8ce7-1215cdc06eab\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 217, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 3c85fee5-2c8b-46c2-a723-6c621ecfb3be, kind: Service { id: 3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7 }, name: Name(\"nexus-3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 215])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e72f50ee-9b20-4af9-becc-46f37972a3b0, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 03f00562-72e1-4d42-8124-11bb07005618, kind: Service { id: e72f50ee-9b20-4af9-becc-46f37972a3b0 }, name: Name(\"nexus-e72f50ee-9b20-4af9-becc-46f37972a3b0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 176, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7fcbadc5-146c-4a16-bbe8-9fe02e4fe911, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: fbaa903b-4740-4365-b35e-68123cc1d3a2, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1693d50c-fcf2-4249-86ae-6e12a08ac868, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 065d24ba-ce47-47d1-9175-26eabb479878, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: bc1764f3-5054-4735-919d-fb5def7e7e42, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a493457b-acb1-4029-bcc4-1d8290fa8759, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 1daa94e0-684d-4a00-9332-60a34c24e94a, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: fdc9e743-1329-4663-9bc9-0b2a07234456, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:49.716184126Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:49.716852748Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:49.716867598Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:49.737254248Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:49.737293929Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:49.756345936Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:49.756382156Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:49.774788382Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:49.774834393Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:49.792373523Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:49.792414353Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b099cd4-61db-4587-84b8-cf9015577106, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:49.80976832Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4b099cd4-61db-4587-84b8-cf9015577106, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:49.809814031Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:49.826888582Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:49.826925723Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4c8e127e-7304-49fa-9de2-79bbde180d94, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:49.845566153Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4c8e127e-7304-49fa-9de2-79bbde180d94, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:49.845603373Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d60f3c38-1add-47dd-baca-a6103e9614ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:49.864259683Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d60f3c38-1add-47dd-baca-a6103e9614ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:49.864307974Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:49.883932931Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:49.883971111Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:49.903548708Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:49.903590258Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bc1764f3-5054-4735-919d-fb5def7e7e42, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:49.922510423Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bc1764f3-5054-4735-919d-fb5def7e7e42, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:49.922545093Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a493457b-acb1-4029-bcc4-1d8290fa8759, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:49.940755885Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a493457b-acb1-4029-bcc4-1d8290fa8759, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:49.940792796Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:49.958824474Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:49.958864515Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:49.97666476Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:49.976713981Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1daa94e0-684d-4a00-9332-60a34c24e94a, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:49.994341872Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1daa94e0-684d-4a00-9332-60a34c24e94a, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:49.994382548Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:50.013346781Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:50.013387029Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:50.032403625Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:50.032441375Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fdc9e743-1329-4663-9bc9-0b2a07234456, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:50.05077925Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fdc9e743-1329-4663-9bc9-0b2a07234456, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:50.05082423Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:50.069040882Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:50.069078343Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:50.087417777Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:50.087454028Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:50.087631351Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:50.087773933Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:50.180622254Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:50.180681646Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:50.180691306Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:50.180720046Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:50.180728256Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:50.180736447Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:50.180742536Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:50.180750547Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:50.180759047Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:50.180765937Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:50.180774357Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:50.180780917Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:50.180789087Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:50.180795937Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:50.180804538Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:50.180812938Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:50.180819538Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:50.180827668Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:50.180840928Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:50.18093336Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"cddb141b-bcca-465d-9ac7-743c671f4990","remote_addr":"[fd00:1122:3344:101::1]:42546","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\", TimeNotSynchronized), (\"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846\", TimeNotSynchronized), (\"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911\", TimeNotSynchronized), (\"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42\", TimeNotSynchronized), (\"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c\", TimeNotSynchronized), (\"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a\", TimeNotSynchronized), (\"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af\", TimeNotSynchronized), (\"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878\", TimeNotSynchronized), (\"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff\", TimeNotSynchronized), (\"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759\", TimeNotSynchronized), (\"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0\", TimeNotSynchronized), (\"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868\", TimeNotSynchronized), (\"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae\", TimeNotSynchronized), (\"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2\", TimeNotSynchronized), (\"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47\", TimeNotSynchronized), (\"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456\", TimeNotSynchronized), (\"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95\", TimeNotSynchronized), (\"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7\", TimeNotSynchronized), (\"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab\", TimeNotSynchronized)]","latency_us":465521,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:50.181124783Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"103.342838ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"cddb141b-bcca-465d-9ac7-743c671f4990\", \"content-length\": \"124\", \"date\": \"Mon, 26 Feb 2024 07:12:49 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"cddb141b-bcca-465d-9ac7-743c671f4990\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: d0d03c23-79c4-4c60-898e-739f188b5615, kind: Service { id: 3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc }, name: Name(\"ntp-3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 196, 59])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 4b099cd4-61db-4587-84b8-cf9015577106, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 4c8e127e-7304-49fa-9de2-79bbde180d94, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: d60f3c38-1add-47dd-baca-a6103e9614ae, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 864c4b26-c35e-42dc-a5b0-3e041205ccbc, kind: Service { id: d60f3c38-1add-47dd-baca-a6103e9614ae }, name: Name(\"external-dns-d60f3c38-1add-47dd-baca-a6103e9614ae\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 7])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: c9ac5e72-089e-4e66-8c75-d1c69a21aee1, kind: Service { id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95 }, name: Name(\"external-dns-740e2473-798e-4bf1-9bf5-7a2c9ce38b95\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 248])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 60944a67-2d69-42d2-8ce7-1215cdc06eab, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 850653ea-9272-4e6c-b3ae-b169328ac41c, kind: Service { id: 60944a67-2d69-42d2-8ce7-1215cdc06eab }, name: Name(\"nexus-60944a67-2d69-42d2-8ce7-1215cdc06eab\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 217, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 3c85fee5-2c8b-46c2-a723-6c621ecfb3be, kind: Service { id: 3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7 }, name: Name(\"nexus-3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 215])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e72f50ee-9b20-4af9-becc-46f37972a3b0, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 03f00562-72e1-4d42-8124-11bb07005618, kind: Service { id: e72f50ee-9b20-4af9-becc-46f37972a3b0 }, name: Name(\"nexus-e72f50ee-9b20-4af9-becc-46f37972a3b0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 176, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7fcbadc5-146c-4a16-bbe8-9fe02e4fe911, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: fbaa903b-4740-4365-b35e-68123cc1d3a2, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1693d50c-fcf2-4249-86ae-6e12a08ac868, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 065d24ba-ce47-47d1-9175-26eabb479878, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: bc1764f3-5054-4735-919d-fb5def7e7e42, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a493457b-acb1-4029-bcc4-1d8290fa8759, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 1daa94e0-684d-4a00-9332-60a34c24e94a, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: fdc9e743-1329-4663-9bc9-0b2a07234456, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:50.285234333Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:50.285868974Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:50.285884505Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:50.30770358Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:50.307764752Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:50.328273884Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:50.328328385Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:50.346351143Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:50.346406119Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:50.363799972Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:50.363857683Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b099cd4-61db-4587-84b8-cf9015577106, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:50.381128668Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4b099cd4-61db-4587-84b8-cf9015577106, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:50.381180979Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:50.398951113Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:50.399012974Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4c8e127e-7304-49fa-9de2-79bbde180d94, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:50.417157995Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4c8e127e-7304-49fa-9de2-79bbde180d94, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:50.417207846Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d60f3c38-1add-47dd-baca-a6103e9614ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:50.434724065Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d60f3c38-1add-47dd-baca-a6103e9614ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:50.434775456Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:50.453333054Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:50.453388695Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:50.471935033Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:50.471986824Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bc1764f3-5054-4735-919d-fb5def7e7e42, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:50.49098625Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bc1764f3-5054-4735-919d-fb5def7e7e42, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:50.491038979Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a493457b-acb1-4029-bcc4-1d8290fa8759, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:50.508730493Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a493457b-acb1-4029-bcc4-1d8290fa8759, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:50.508778224Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:50.52665818Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:50.526716861Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:50.544887934Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:50.544937513Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1daa94e0-684d-4a00-9332-60a34c24e94a, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:50.56342438Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1daa94e0-684d-4a00-9332-60a34c24e94a, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:50.563480021Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:50.582399865Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:50.582455346Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:50.602535481Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:50.602592352Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fdc9e743-1329-4663-9bc9-0b2a07234456, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:50.62113173Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fdc9e743-1329-4663-9bc9-0b2a07234456, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:50.621188431Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:50.639842801Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:50.639896671Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:50.657541623Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:50.657599989Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:50.657855909Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:50.658018312Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:50.748939818Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:50.74901444Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:50.74902585Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:50.74903261Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:50.74903939Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:50.74904907Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:50.74905727Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:50.749063571Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:50.749071641Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:50.749080201Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:50.749088511Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:50.749099041Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:50.749105621Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:50.749113611Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:50.749120112Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:50.749138772Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:50.749146712Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:50.749152842Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:50.749159412Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:50.749242974Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"c1541f66-490e-4023-8c56-6f63c0d96ed5","remote_addr":"[fd00:1122:3344:101::1]:42546","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868\", TimeNotSynchronized), (\"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae\", TimeNotSynchronized), (\"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95\", TimeNotSynchronized), (\"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911\", TimeNotSynchronized), (\"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2\", TimeNotSynchronized), (\"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878\", TimeNotSynchronized), (\"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab\", TimeNotSynchronized), (\"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\", TimeNotSynchronized), (\"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0\", TimeNotSynchronized), (\"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42\", TimeNotSynchronized), (\"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47\", TimeNotSynchronized), (\"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a\", TimeNotSynchronized), (\"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff\", TimeNotSynchronized), (\"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7\", TimeNotSynchronized), (\"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c\", TimeNotSynchronized), (\"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846\", TimeNotSynchronized), (\"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af\", TimeNotSynchronized), (\"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456\", TimeNotSynchronized), (\"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759\", TimeNotSynchronized)]","latency_us":464826,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:50.749448447Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"156.088196ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c1541f66-490e-4023-8c56-6f63c0d96ed5\", \"content-length\": \"124\", \"date\": \"Mon, 26 Feb 2024 07:12:50 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c1541f66-490e-4023-8c56-6f63c0d96ed5\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: d0d03c23-79c4-4c60-898e-739f188b5615, kind: Service { id: 3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc }, name: Name(\"ntp-3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 196, 59])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 4b099cd4-61db-4587-84b8-cf9015577106, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 4c8e127e-7304-49fa-9de2-79bbde180d94, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: d60f3c38-1add-47dd-baca-a6103e9614ae, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 864c4b26-c35e-42dc-a5b0-3e041205ccbc, kind: Service { id: d60f3c38-1add-47dd-baca-a6103e9614ae }, name: Name(\"external-dns-d60f3c38-1add-47dd-baca-a6103e9614ae\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 7])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: c9ac5e72-089e-4e66-8c75-d1c69a21aee1, kind: Service { id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95 }, name: Name(\"external-dns-740e2473-798e-4bf1-9bf5-7a2c9ce38b95\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 248])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 60944a67-2d69-42d2-8ce7-1215cdc06eab, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 850653ea-9272-4e6c-b3ae-b169328ac41c, kind: Service { id: 60944a67-2d69-42d2-8ce7-1215cdc06eab }, name: Name(\"nexus-60944a67-2d69-42d2-8ce7-1215cdc06eab\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 217, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 3c85fee5-2c8b-46c2-a723-6c621ecfb3be, kind: Service { id: 3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7 }, name: Name(\"nexus-3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 215])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e72f50ee-9b20-4af9-becc-46f37972a3b0, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 03f00562-72e1-4d42-8124-11bb07005618, kind: Service { id: e72f50ee-9b20-4af9-becc-46f37972a3b0 }, name: Name(\"nexus-e72f50ee-9b20-4af9-becc-46f37972a3b0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 176, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7fcbadc5-146c-4a16-bbe8-9fe02e4fe911, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: fbaa903b-4740-4365-b35e-68123cc1d3a2, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1693d50c-fcf2-4249-86ae-6e12a08ac868, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 065d24ba-ce47-47d1-9175-26eabb479878, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: bc1764f3-5054-4735-919d-fb5def7e7e42, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a493457b-acb1-4029-bcc4-1d8290fa8759, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 1daa94e0-684d-4a00-9332-60a34c24e94a, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: fdc9e743-1329-4663-9bc9-0b2a07234456, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:50.906231238Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:50.90687428Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:50.906890677Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:50.928674525Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:50.928729146Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:50.947944966Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:50.947995956Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:50.966433742Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:50.966481983Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:50.984326478Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:50.984381079Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b099cd4-61db-4587-84b8-cf9015577106, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.004328812Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4b099cd4-61db-4587-84b8-cf9015577106, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.004384983Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.023701854Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.023756566Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4c8e127e-7304-49fa-9de2-79bbde180d94, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.041696132Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4c8e127e-7304-49fa-9de2-79bbde180d94, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.041744303Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d60f3c38-1add-47dd-baca-a6103e9614ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.059220402Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d60f3c38-1add-47dd-baca-a6103e9614ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.059267603Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.078017994Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.078070985Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.097225104Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.097274205Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bc1764f3-5054-4735-919d-fb5def7e7e42, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.118172574Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bc1764f3-5054-4735-919d-fb5def7e7e42, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.118219045Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a493457b-acb1-4029-bcc4-1d8290fa8759, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.136175792Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a493457b-acb1-4029-bcc4-1d8290fa8759, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.136226913Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.155486924Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.155533604Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.174096792Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.174146113Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1daa94e0-684d-4a00-9332-60a34c24e94a, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.193559427Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1daa94e0-684d-4a00-9332-60a34c24e94a, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.193615617Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.212352879Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.21240336Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.230957437Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.231005508Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fdc9e743-1329-4663-9bc9-0b2a07234456, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.249542786Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fdc9e743-1329-4663-9bc9-0b2a07234456, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.249600247Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.26842335Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.26847312Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.287338154Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.287391595Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.287622849Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.287769732Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:51.385186963Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:51.385243974Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:51.385268175Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:51.385283205Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:51.385294005Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:51.385301725Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:51.385307475Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:51.385315425Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:51.385323366Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:51.385330906Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:51.385338306Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:51.385352236Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:51.385368877Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:51.385376797Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:51.385384447Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:51.385391817Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:51.385396307Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:51.385401837Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:51.385405957Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.385495559Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"2d5f5426-855e-4733-942a-b1d79d42a225","remote_addr":"[fd00:1122:3344:101::1]:42546","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868\", TimeNotSynchronized), (\"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2\", TimeNotSynchronized), (\"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759\", TimeNotSynchronized), (\"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95\", TimeNotSynchronized), (\"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911\", TimeNotSynchronized), (\"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42\", TimeNotSynchronized), (\"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47\", TimeNotSynchronized), (\"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456\", TimeNotSynchronized), (\"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7\", TimeNotSynchronized), (\"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af\", TimeNotSynchronized), (\"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae\", TimeNotSynchronized), (\"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab\", TimeNotSynchronized), (\"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a\", TimeNotSynchronized), (\"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0\", TimeNotSynchronized), (\"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff\", TimeNotSynchronized), (\"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878\", TimeNotSynchronized), (\"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c\", TimeNotSynchronized), (\"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\", TimeNotSynchronized), (\"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846\", TimeNotSynchronized)]","latency_us":480111,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:51.385715423Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"185.22247ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2d5f5426-855e-4733-942a-b1d79d42a225\", \"content-length\": \"124\", \"date\": \"Mon, 26 Feb 2024 07:12:50 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"2d5f5426-855e-4733-942a-b1d79d42a225\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: d0d03c23-79c4-4c60-898e-739f188b5615, kind: Service { id: 3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc }, name: Name(\"ntp-3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 196, 59])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 4b099cd4-61db-4587-84b8-cf9015577106, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 4c8e127e-7304-49fa-9de2-79bbde180d94, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: d60f3c38-1add-47dd-baca-a6103e9614ae, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 864c4b26-c35e-42dc-a5b0-3e041205ccbc, kind: Service { id: d60f3c38-1add-47dd-baca-a6103e9614ae }, name: Name(\"external-dns-d60f3c38-1add-47dd-baca-a6103e9614ae\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 7])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: c9ac5e72-089e-4e66-8c75-d1c69a21aee1, kind: Service { id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95 }, name: Name(\"external-dns-740e2473-798e-4bf1-9bf5-7a2c9ce38b95\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 248])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 60944a67-2d69-42d2-8ce7-1215cdc06eab, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 850653ea-9272-4e6c-b3ae-b169328ac41c, kind: Service { id: 60944a67-2d69-42d2-8ce7-1215cdc06eab }, name: Name(\"nexus-60944a67-2d69-42d2-8ce7-1215cdc06eab\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 217, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 3c85fee5-2c8b-46c2-a723-6c621ecfb3be, kind: Service { id: 3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7 }, name: Name(\"nexus-3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 215])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e72f50ee-9b20-4af9-becc-46f37972a3b0, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 03f00562-72e1-4d42-8124-11bb07005618, kind: Service { id: e72f50ee-9b20-4af9-becc-46f37972a3b0 }, name: Name(\"nexus-e72f50ee-9b20-4af9-becc-46f37972a3b0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 176, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7fcbadc5-146c-4a16-bbe8-9fe02e4fe911, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: fbaa903b-4740-4365-b35e-68123cc1d3a2, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1693d50c-fcf2-4249-86ae-6e12a08ac868, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 065d24ba-ce47-47d1-9175-26eabb479878, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: bc1764f3-5054-4735-919d-fb5def7e7e42, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a493457b-acb1-4029-bcc4-1d8290fa8759, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 1daa94e0-684d-4a00-9332-60a34c24e94a, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: fdc9e743-1329-4663-9bc9-0b2a07234456, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.571431265Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.572080366Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.572094991Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.593938153Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.594015654Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.615529224Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.615592655Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.63509789Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.635150631Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.654625745Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.654682226Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b099cd4-61db-4587-84b8-cf9015577106, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.674321884Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4b099cd4-61db-4587-84b8-cf9015577106, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.674377404Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.694173244Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.694224405Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4c8e127e-7304-49fa-9de2-79bbde180d94, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.714411102Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4c8e127e-7304-49fa-9de2-79bbde180d94, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.714461453Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d60f3c38-1add-47dd-baca-a6103e9614ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.732439691Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d60f3c38-1add-47dd-baca-a6103e9614ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.732496052Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.752563166Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.752629131Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.772735153Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.772791714Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bc1764f3-5054-4735-919d-fb5def7e7e42, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.792157496Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bc1764f3-5054-4735-919d-fb5def7e7e42, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.792212087Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a493457b-acb1-4029-bcc4-1d8290fa8759, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.8105051Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a493457b-acb1-4029-bcc4-1d8290fa8759, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.810562411Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.829648829Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.82969645Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.848481911Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.848534112Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1daa94e0-684d-4a00-9332-60a34c24e94a, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.867318038Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1daa94e0-684d-4a00-9332-60a34c24e94a, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.867373865Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.885966634Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.886031865Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.904834637Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.904886538Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fdc9e743-1329-4663-9bc9-0b2a07234456, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.923849794Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fdc9e743-1329-4663-9bc9-0b2a07234456, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.923903194Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.943361098Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.943411679Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.962028898Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.962094089Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.962293683Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:51.962431155Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:52.056883565Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:52.056960026Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:52.056982067Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:52.056987267Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:52.056995007Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:52.057003287Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:52.057012987Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:52.057021697Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:52.057032067Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:52.057054418Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:52.057061218Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:52.057068538Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:52.057075168Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:52.057091028Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:52.057098658Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:52.057103078Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:52.057108549Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:52.057112929Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:52.057117369Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:52.05719941Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"743d2a2b-0b5e-48aa-a9fa-db90d4650921","remote_addr":"[fd00:1122:3344:101::1]:42546","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2\", TimeNotSynchronized), (\"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae\", TimeNotSynchronized), (\"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911\", TimeNotSynchronized), (\"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759\", TimeNotSynchronized), (\"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7\", TimeNotSynchronized), (\"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456\", TimeNotSynchronized), (\"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af\", TimeNotSynchronized), (\"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff\", TimeNotSynchronized), (\"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a\", TimeNotSynchronized), (\"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab\", TimeNotSynchronized), (\"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\", TimeNotSynchronized), (\"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878\", TimeNotSynchronized), (\"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0\", TimeNotSynchronized), (\"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846\", TimeNotSynchronized), (\"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95\", TimeNotSynchronized), (\"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42\", TimeNotSynchronized), (\"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868\", TimeNotSynchronized), (\"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47\", TimeNotSynchronized), (\"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c\", TimeNotSynchronized)]","latency_us":486630,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:52.057416734Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"147.250521ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"743d2a2b-0b5e-48aa-a9fa-db90d4650921\", \"content-length\": \"124\", \"date\": \"Mon, 26 Feb 2024 07:12:52 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"743d2a2b-0b5e-48aa-a9fa-db90d4650921\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: d0d03c23-79c4-4c60-898e-739f188b5615, kind: Service { id: 3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc }, name: Name(\"ntp-3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 196, 59])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 4b099cd4-61db-4587-84b8-cf9015577106, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 4c8e127e-7304-49fa-9de2-79bbde180d94, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: d60f3c38-1add-47dd-baca-a6103e9614ae, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 864c4b26-c35e-42dc-a5b0-3e041205ccbc, kind: Service { id: d60f3c38-1add-47dd-baca-a6103e9614ae }, name: Name(\"external-dns-d60f3c38-1add-47dd-baca-a6103e9614ae\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 7])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: c9ac5e72-089e-4e66-8c75-d1c69a21aee1, kind: Service { id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95 }, name: Name(\"external-dns-740e2473-798e-4bf1-9bf5-7a2c9ce38b95\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 248])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 60944a67-2d69-42d2-8ce7-1215cdc06eab, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 850653ea-9272-4e6c-b3ae-b169328ac41c, kind: Service { id: 60944a67-2d69-42d2-8ce7-1215cdc06eab }, name: Name(\"nexus-60944a67-2d69-42d2-8ce7-1215cdc06eab\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 217, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 3c85fee5-2c8b-46c2-a723-6c621ecfb3be, kind: Service { id: 3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7 }, name: Name(\"nexus-3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 215])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e72f50ee-9b20-4af9-becc-46f37972a3b0, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 03f00562-72e1-4d42-8124-11bb07005618, kind: Service { id: e72f50ee-9b20-4af9-becc-46f37972a3b0 }, name: Name(\"nexus-e72f50ee-9b20-4af9-becc-46f37972a3b0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 176, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7fcbadc5-146c-4a16-bbe8-9fe02e4fe911, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: fbaa903b-4740-4365-b35e-68123cc1d3a2, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1693d50c-fcf2-4249-86ae-6e12a08ac868, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 065d24ba-ce47-47d1-9175-26eabb479878, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: bc1764f3-5054-4735-919d-fb5def7e7e42, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a493457b-acb1-4029-bcc4-1d8290fa8759, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 1daa94e0-684d-4a00-9332-60a34c24e94a, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: fdc9e743-1329-4663-9bc9-0b2a07234456, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:52.205183806Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:52.205905828Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:52.205925889Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:52.226649255Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:52.226711766Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:52.246582387Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:52.246639759Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:52.266634732Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:52.266695213Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:52.286806328Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:52.289377364Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b099cd4-61db-4587-84b8-cf9015577106, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:52.306117422Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4b099cd4-61db-4587-84b8-cf9015577106, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:52.30617048Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:52.324369662Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:52.324435703Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4c8e127e-7304-49fa-9de2-79bbde180d94, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:52.345355253Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4c8e127e-7304-49fa-9de2-79bbde180d94, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:52.345411294Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d60f3c38-1add-47dd-baca-a6103e9614ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:52.365419288Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d60f3c38-1add-47dd-baca-a6103e9614ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:52.365475039Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:52.386267656Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:52.386328157Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:52.406899411Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:52.406960132Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bc1764f3-5054-4735-919d-fb5def7e7e42, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:52.426759252Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bc1764f3-5054-4735-919d-fb5def7e7e42, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:52.429245516Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a493457b-acb1-4029-bcc4-1d8290fa8759, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:52.447767113Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a493457b-acb1-4029-bcc4-1d8290fa8759, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:52.447823594Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:52.467262978Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:52.467326979Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:52.487100248Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:52.487163229Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1daa94e0-684d-4a00-9332-60a34c24e94a, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:52.507438608Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1daa94e0-684d-4a00-9332-60a34c24e94a, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:52.507503209Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:52.526426863Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:52.526475404Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:52.547482086Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:52.547549067Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fdc9e743-1329-4663-9bc9-0b2a07234456, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:52.565465093Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fdc9e743-1329-4663-9bc9-0b2a07234456, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:52.565517474Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:52.58452822Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:52.584581951Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:52.602811553Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:52.602883125Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:52.603091228Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:52.603246661Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:52.696718063Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:52.696818035Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:52.696826785Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:52.696838515Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:52.696847905Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:52.696855625Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:52.696861036Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:52.696882146Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:52.696891406Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:52.696900186Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:52.696911106Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:52.696919737Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:52.696928417Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:52.696934757Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:52.696939797Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:52.696945047Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:52.696951537Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:52.696958247Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:52.696965597Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:52.697034909Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"0f8df7e4-82e8-4598-95c5-0a24073b7dd1","remote_addr":"[fd00:1122:3344:101::1]:42546","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2\", TimeNotSynchronized), (\"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7\", TimeNotSynchronized), (\"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878\", TimeNotSynchronized), (\"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846\", TimeNotSynchronized), (\"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911\", TimeNotSynchronized), (\"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47\", TimeNotSynchronized), (\"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a\", TimeNotSynchronized), (\"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae\", TimeNotSynchronized), (\"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95\", TimeNotSynchronized), (\"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c\", TimeNotSynchronized), (\"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff\", TimeNotSynchronized), (\"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868\", TimeNotSynchronized), (\"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af\", TimeNotSynchronized), (\"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab\", TimeNotSynchronized), (\"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42\", TimeNotSynchronized), (\"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759\", TimeNotSynchronized), (\"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0\", TimeNotSynchronized), (\"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\", TimeNotSynchronized), (\"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456\", TimeNotSynchronized)]","latency_us":492684,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:52.697243082Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"250.283773ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0f8df7e4-82e8-4598-95c5-0a24073b7dd1\", \"content-length\": \"124\", \"date\": \"Mon, 26 Feb 2024 07:12:52 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"0f8df7e4-82e8-4598-95c5-0a24073b7dd1\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: d0d03c23-79c4-4c60-898e-739f188b5615, kind: Service { id: 3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc }, name: Name(\"ntp-3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 196, 59])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 4b099cd4-61db-4587-84b8-cf9015577106, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 4c8e127e-7304-49fa-9de2-79bbde180d94, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: d60f3c38-1add-47dd-baca-a6103e9614ae, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 864c4b26-c35e-42dc-a5b0-3e041205ccbc, kind: Service { id: d60f3c38-1add-47dd-baca-a6103e9614ae }, name: Name(\"external-dns-d60f3c38-1add-47dd-baca-a6103e9614ae\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 7])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: c9ac5e72-089e-4e66-8c75-d1c69a21aee1, kind: Service { id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95 }, name: Name(\"external-dns-740e2473-798e-4bf1-9bf5-7a2c9ce38b95\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 248])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 60944a67-2d69-42d2-8ce7-1215cdc06eab, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 850653ea-9272-4e6c-b3ae-b169328ac41c, kind: Service { id: 60944a67-2d69-42d2-8ce7-1215cdc06eab }, name: Name(\"nexus-60944a67-2d69-42d2-8ce7-1215cdc06eab\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 217, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 3c85fee5-2c8b-46c2-a723-6c621ecfb3be, kind: Service { id: 3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7 }, name: Name(\"nexus-3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 215])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e72f50ee-9b20-4af9-becc-46f37972a3b0, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 03f00562-72e1-4d42-8124-11bb07005618, kind: Service { id: e72f50ee-9b20-4af9-becc-46f37972a3b0 }, name: Name(\"nexus-e72f50ee-9b20-4af9-becc-46f37972a3b0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 176, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7fcbadc5-146c-4a16-bbe8-9fe02e4fe911, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: fbaa903b-4740-4365-b35e-68123cc1d3a2, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1693d50c-fcf2-4249-86ae-6e12a08ac868, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 065d24ba-ce47-47d1-9175-26eabb479878, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: bc1764f3-5054-4735-919d-fb5def7e7e42, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a493457b-acb1-4029-bcc4-1d8290fa8759, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 1daa94e0-684d-4a00-9332-60a34c24e94a, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: fdc9e743-1329-4663-9bc9-0b2a07234456, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:52.94779013Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:52.948429501Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:52.948444262Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:52.969446183Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:52.969511614Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:52.990379943Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:52.990436734Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:53.008805008Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:53.00887311Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:53.02916554Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:53.029219499Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b099cd4-61db-4587-84b8-cf9015577106, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:53.046939662Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4b099cd4-61db-4587-84b8-cf9015577106, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:53.047002533Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:53.064602575Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:53.064663156Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4c8e127e-7304-49fa-9de2-79bbde180d94, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:53.085160548Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4c8e127e-7304-49fa-9de2-79bbde180d94, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:53.085234509Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d60f3c38-1add-47dd-baca-a6103e9614ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:53.104802385Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d60f3c38-1add-47dd-baca-a6103e9614ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:53.104858666Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:53.124762818Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:53.124821999Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:53.144612849Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:53.144665859Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bc1764f3-5054-4735-919d-fb5def7e7e42, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:53.16563643Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bc1764f3-5054-4735-919d-fb5def7e7e42, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:53.165687441Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a493457b-acb1-4029-bcc4-1d8290fa8759, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:53.184078426Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a493457b-acb1-4029-bcc4-1d8290fa8759, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:53.184130757Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:53.202590763Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:53.202641274Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:53.220949408Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:53.221022329Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1daa94e0-684d-4a00-9332-60a34c24e94a, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:53.239611332Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1daa94e0-684d-4a00-9332-60a34c24e94a, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:53.239673759Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:53.257762058Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:53.25781875Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:53.275520262Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:53.275576113Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fdc9e743-1329-4663-9bc9-0b2a07234456, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:53.293478022Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fdc9e743-1329-4663-9bc9-0b2a07234456, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:53.293537791Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:53.311411176Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:53.311469678Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:53.329832522Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:53.329911444Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:53.330141978Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:53.330303911Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:53.42530932Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:53.425391471Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:53.425399561Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:53.425407041Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:53.425421482Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:53.425427512Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:53.425435272Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:53.425446412Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:53.425452412Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:53.425459682Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:53.425465533Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:53.425472802Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:53.425478533Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:53.425485843Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:53.425491543Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:53.425495903Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:53.425501563Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:53.425506283Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:53.425510733Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:53.425599685Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"00af2c6c-8a3a-4538-acad-5796c9b081fc","remote_addr":"[fd00:1122:3344:101::1]:42546","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0\", TimeNotSynchronized), (\"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab\", TimeNotSynchronized), (\"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\", TimeNotSynchronized), (\"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7\", TimeNotSynchronized), (\"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2\", TimeNotSynchronized), (\"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868\", TimeNotSynchronized), (\"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae\", TimeNotSynchronized), (\"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878\", TimeNotSynchronized), (\"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47\", TimeNotSynchronized), (\"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff\", TimeNotSynchronized), (\"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42\", TimeNotSynchronized), (\"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af\", TimeNotSynchronized), (\"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911\", TimeNotSynchronized), (\"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95\", TimeNotSynchronized), (\"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c\", TimeNotSynchronized), (\"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a\", TimeNotSynchronized), (\"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456\", TimeNotSynchronized), (\"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759\", TimeNotSynchronized), (\"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846\", TimeNotSynchronized)]","latency_us":478662,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:53.425843309Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"329.840457ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"00af2c6c-8a3a-4538-acad-5796c9b081fc\", \"content-length\": \"124\", \"date\": \"Mon, 26 Feb 2024 07:12:53 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"00af2c6c-8a3a-4538-acad-5796c9b081fc\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: d0d03c23-79c4-4c60-898e-739f188b5615, kind: Service { id: 3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc }, name: Name(\"ntp-3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 196, 59])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 4b099cd4-61db-4587-84b8-cf9015577106, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 4c8e127e-7304-49fa-9de2-79bbde180d94, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: d60f3c38-1add-47dd-baca-a6103e9614ae, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 864c4b26-c35e-42dc-a5b0-3e041205ccbc, kind: Service { id: d60f3c38-1add-47dd-baca-a6103e9614ae }, name: Name(\"external-dns-d60f3c38-1add-47dd-baca-a6103e9614ae\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 7])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: c9ac5e72-089e-4e66-8c75-d1c69a21aee1, kind: Service { id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95 }, name: Name(\"external-dns-740e2473-798e-4bf1-9bf5-7a2c9ce38b95\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 248])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 60944a67-2d69-42d2-8ce7-1215cdc06eab, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 850653ea-9272-4e6c-b3ae-b169328ac41c, kind: Service { id: 60944a67-2d69-42d2-8ce7-1215cdc06eab }, name: Name(\"nexus-60944a67-2d69-42d2-8ce7-1215cdc06eab\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 217, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 3c85fee5-2c8b-46c2-a723-6c621ecfb3be, kind: Service { id: 3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7 }, name: Name(\"nexus-3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 215])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e72f50ee-9b20-4af9-becc-46f37972a3b0, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 03f00562-72e1-4d42-8124-11bb07005618, kind: Service { id: e72f50ee-9b20-4af9-becc-46f37972a3b0 }, name: Name(\"nexus-e72f50ee-9b20-4af9-becc-46f37972a3b0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 176, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7fcbadc5-146c-4a16-bbe8-9fe02e4fe911, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: fbaa903b-4740-4365-b35e-68123cc1d3a2, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1693d50c-fcf2-4249-86ae-6e12a08ac868, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 065d24ba-ce47-47d1-9175-26eabb479878, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: bc1764f3-5054-4735-919d-fb5def7e7e42, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a493457b-acb1-4029-bcc4-1d8290fa8759, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 1daa94e0-684d-4a00-9332-60a34c24e94a, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: fdc9e743-1329-4663-9bc9-0b2a07234456, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:53.756204209Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:53.756843719Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:53.756862999Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:53.779999108Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:53.780055199Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:53.801496958Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:53.801552109Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:53.821765916Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:53.821836498Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:53.841158489Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:53.84122674Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b099cd4-61db-4587-84b8-cf9015577106, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:53.860920127Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4b099cd4-61db-4587-84b8-cf9015577106, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:53.860971479Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:53.879075729Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:53.87913628Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4c8e127e-7304-49fa-9de2-79bbde180d94, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:53.898993471Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4c8e127e-7304-49fa-9de2-79bbde180d94, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:53.899045522Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d60f3c38-1add-47dd-baca-a6103e9614ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:53.916844477Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d60f3c38-1add-47dd-baca-a6103e9614ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:53.916897098Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:53.936553125Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:53.936605086Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:53.956664079Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:53.956724462Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bc1764f3-5054-4735-919d-fb5def7e7e42, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:53.975243899Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bc1764f3-5054-4735-919d-fb5def7e7e42, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:53.97530329Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a493457b-acb1-4029-bcc4-1d8290fa8759, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:53.995928115Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a493457b-acb1-4029-bcc4-1d8290fa8759, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:53.995982865Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:54.015616403Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:54.015674644Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:54.036077484Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:54.036136455Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1daa94e0-684d-4a00-9332-60a34c24e94a, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:54.056247921Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1daa94e0-684d-4a00-9332-60a34c24e94a, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:54.056299171Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:54.07545955Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:54.075511391Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:54.095064687Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:54.095129158Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fdc9e743-1329-4663-9bc9-0b2a07234456, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:54.11506945Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fdc9e743-1329-4663-9bc9-0b2a07234456, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:54.115130911Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:54.134071926Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:54.134123097Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:54.153867366Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:54.153936727Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:54.154132192Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:54.154268903Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:54.250188208Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:54.25028349Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:54.25029145Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:54.25029803Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:54.25030562Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:54.25031309Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:54.250320911Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:54.250327411Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:54.250334471Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:54.250340331Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:54.250347351Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:54.250355191Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:54.250359621Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:54.250365631Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:54.250370151Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:54.250374261Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:54.250390782Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:54.250398212Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:54.250404012Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:54.250471523Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"90a79617-5b37-45d1-b11f-ffa3a42d942c","remote_addr":"[fd00:1122:3344:101::1]:42546","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846\", TimeNotSynchronized), (\"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff\", TimeNotSynchronized), (\"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7\", TimeNotSynchronized), (\"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868\", TimeNotSynchronized), (\"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47\", TimeNotSynchronized), (\"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456\", TimeNotSynchronized), (\"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0\", TimeNotSynchronized), (\"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c\", TimeNotSynchronized), (\"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878\", TimeNotSynchronized), (\"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae\", TimeNotSynchronized), (\"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2\", TimeNotSynchronized), (\"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab\", TimeNotSynchronized), (\"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911\", TimeNotSynchronized), (\"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759\", TimeNotSynchronized), (\"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a\", TimeNotSynchronized), (\"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95\", TimeNotSynchronized), (\"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\", TimeNotSynchronized), (\"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af\", TimeNotSynchronized), (\"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42\", TimeNotSynchronized)]","latency_us":495167,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:54.250706887Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"448.33611ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"90a79617-5b37-45d1-b11f-ffa3a42d942c\", \"content-length\": \"124\", \"date\": \"Mon, 26 Feb 2024 07:12:53 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"90a79617-5b37-45d1-b11f-ffa3a42d942c\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: d0d03c23-79c4-4c60-898e-739f188b5615, kind: Service { id: 3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc }, name: Name(\"ntp-3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 196, 59])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 4b099cd4-61db-4587-84b8-cf9015577106, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 4c8e127e-7304-49fa-9de2-79bbde180d94, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: d60f3c38-1add-47dd-baca-a6103e9614ae, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 864c4b26-c35e-42dc-a5b0-3e041205ccbc, kind: Service { id: d60f3c38-1add-47dd-baca-a6103e9614ae }, name: Name(\"external-dns-d60f3c38-1add-47dd-baca-a6103e9614ae\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 7])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: c9ac5e72-089e-4e66-8c75-d1c69a21aee1, kind: Service { id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95 }, name: Name(\"external-dns-740e2473-798e-4bf1-9bf5-7a2c9ce38b95\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 248])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 60944a67-2d69-42d2-8ce7-1215cdc06eab, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 850653ea-9272-4e6c-b3ae-b169328ac41c, kind: Service { id: 60944a67-2d69-42d2-8ce7-1215cdc06eab }, name: Name(\"nexus-60944a67-2d69-42d2-8ce7-1215cdc06eab\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 217, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 3c85fee5-2c8b-46c2-a723-6c621ecfb3be, kind: Service { id: 3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7 }, name: Name(\"nexus-3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 215])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e72f50ee-9b20-4af9-becc-46f37972a3b0, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 03f00562-72e1-4d42-8124-11bb07005618, kind: Service { id: e72f50ee-9b20-4af9-becc-46f37972a3b0 }, name: Name(\"nexus-e72f50ee-9b20-4af9-becc-46f37972a3b0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 176, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7fcbadc5-146c-4a16-bbe8-9fe02e4fe911, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: fbaa903b-4740-4365-b35e-68123cc1d3a2, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1693d50c-fcf2-4249-86ae-6e12a08ac868, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 065d24ba-ce47-47d1-9175-26eabb479878, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: bc1764f3-5054-4735-919d-fb5def7e7e42, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a493457b-acb1-4029-bcc4-1d8290fa8759, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 1daa94e0-684d-4a00-9332-60a34c24e94a, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: fdc9e743-1329-4663-9bc9-0b2a07234456, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:54.699772174Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:54.701018056Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:54.701041006Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:54.722971964Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:54.723038805Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:54.745023303Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:54.745090585Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:54.766530974Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:54.766588735Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:54.787119547Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:54.787172408Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b099cd4-61db-4587-84b8-cf9015577106, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:54.806850706Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4b099cd4-61db-4587-84b8-cf9015577106, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:54.806908307Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:54.826063346Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:54.826127387Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4c8e127e-7304-49fa-9de2-79bbde180d94, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:54.845102062Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4c8e127e-7304-49fa-9de2-79bbde180d94, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:54.845153713Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d60f3c38-1add-47dd-baca-a6103e9614ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:54.86309689Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d60f3c38-1add-47dd-baca-a6103e9614ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:54.863159582Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:54.882797909Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:54.88287298Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:54.90155946Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:54.901620541Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bc1764f3-5054-4735-919d-fb5def7e7e42, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:54.919844183Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bc1764f3-5054-4735-919d-fb5def7e7e42, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:54.919902194Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a493457b-acb1-4029-bcc4-1d8290fa8759, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:54.938720287Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a493457b-acb1-4029-bcc4-1d8290fa8759, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:54.938779268Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:54.956762186Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:54.956820997Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:54.974592501Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:54.974647952Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1daa94e0-684d-4a00-9332-60a34c24e94a, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:54.994384711Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1daa94e0-684d-4a00-9332-60a34c24e94a, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:54.994443112Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:55.015186418Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:55.015243759Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:55.035286934Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:55.035347095Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fdc9e743-1329-4663-9bc9-0b2a07234456, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:55.056869485Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fdc9e743-1329-4663-9bc9-0b2a07234456, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:55.056931826Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:55.076411591Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:55.076464392Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:55.095221593Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:55.095295114Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:55.095490758Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:55.09563619Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:55.191541695Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:55.191616217Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:55.191632037Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:55.191639867Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:55.191652537Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:55.191658527Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:55.191664988Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:55.191671028Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:55.191678378Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:55.191685758Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:55.191692888Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:55.191698828Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:55.191712578Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:55.191736829Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:55.191743099Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:55.191748869Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:55.191753239Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:55.191762779Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:55.191768309Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:55.191841491Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"095b3d53-25e0-43ae-bce5-029c3d72a3cf","remote_addr":"[fd00:1122:3344:101::1]:42546","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868\", TimeNotSynchronized), (\"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af\", TimeNotSynchronized), (\"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7\", TimeNotSynchronized), (\"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\", TimeNotSynchronized), (\"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae\", TimeNotSynchronized), (\"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95\", TimeNotSynchronized), (\"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2\", TimeNotSynchronized), (\"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a\", TimeNotSynchronized), (\"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878\", TimeNotSynchronized), (\"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47\", TimeNotSynchronized), (\"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab\", TimeNotSynchronized), (\"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0\", TimeNotSynchronized), (\"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42\", TimeNotSynchronized), (\"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c\", TimeNotSynchronized), (\"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911\", TimeNotSynchronized), (\"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759\", TimeNotSynchronized), (\"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846\", TimeNotSynchronized), (\"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff\", TimeNotSynchronized), (\"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456\", TimeNotSynchronized)]","latency_us":492936,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:55.192068024Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"246.663463ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"095b3d53-25e0-43ae-bce5-029c3d72a3cf\", \"content-length\": \"124\", \"date\": \"Mon, 26 Feb 2024 07:12:54 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"095b3d53-25e0-43ae-bce5-029c3d72a3cf\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: d0d03c23-79c4-4c60-898e-739f188b5615, kind: Service { id: 3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc }, name: Name(\"ntp-3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 196, 59])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 4b099cd4-61db-4587-84b8-cf9015577106, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 4c8e127e-7304-49fa-9de2-79bbde180d94, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: d60f3c38-1add-47dd-baca-a6103e9614ae, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 864c4b26-c35e-42dc-a5b0-3e041205ccbc, kind: Service { id: d60f3c38-1add-47dd-baca-a6103e9614ae }, name: Name(\"external-dns-d60f3c38-1add-47dd-baca-a6103e9614ae\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 7])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: c9ac5e72-089e-4e66-8c75-d1c69a21aee1, kind: Service { id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95 }, name: Name(\"external-dns-740e2473-798e-4bf1-9bf5-7a2c9ce38b95\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 248])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 60944a67-2d69-42d2-8ce7-1215cdc06eab, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 850653ea-9272-4e6c-b3ae-b169328ac41c, kind: Service { id: 60944a67-2d69-42d2-8ce7-1215cdc06eab }, name: Name(\"nexus-60944a67-2d69-42d2-8ce7-1215cdc06eab\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 217, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 3c85fee5-2c8b-46c2-a723-6c621ecfb3be, kind: Service { id: 3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7 }, name: Name(\"nexus-3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 215])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e72f50ee-9b20-4af9-becc-46f37972a3b0, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 03f00562-72e1-4d42-8124-11bb07005618, kind: Service { id: e72f50ee-9b20-4af9-becc-46f37972a3b0 }, name: Name(\"nexus-e72f50ee-9b20-4af9-becc-46f37972a3b0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 176, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7fcbadc5-146c-4a16-bbe8-9fe02e4fe911, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: fbaa903b-4740-4365-b35e-68123cc1d3a2, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1693d50c-fcf2-4249-86ae-6e12a08ac868, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 065d24ba-ce47-47d1-9175-26eabb479878, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: bc1764f3-5054-4735-919d-fb5def7e7e42, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a493457b-acb1-4029-bcc4-1d8290fa8759, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 1daa94e0-684d-4a00-9332-60a34c24e94a, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: fdc9e743-1329-4663-9bc9-0b2a07234456, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:55.439797543Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:55.440424874Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:55.440442044Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:55.462054376Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:55.462121037Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:55.482039149Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:55.482103331Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:55.502602523Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:55.502662284Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:55.523155866Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:55.523212927Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b099cd4-61db-4587-84b8-cf9015577106, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:55.541642313Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4b099cd4-61db-4587-84b8-cf9015577106, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:55.541703084Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:55.561694057Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:55.561745528Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4c8e127e-7304-49fa-9de2-79bbde180d94, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:55.581143831Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4c8e127e-7304-49fa-9de2-79bbde180d94, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:55.581201502Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d60f3c38-1add-47dd-baca-a6103e9614ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:55.600195888Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d60f3c38-1add-47dd-baca-a6103e9614ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:55.600248374Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:55.620347944Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:55.620402045Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:55.638686598Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:55.638750639Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bc1764f3-5054-4735-919d-fb5def7e7e42, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:55.658829404Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bc1764f3-5054-4735-919d-fb5def7e7e42, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:55.658882785Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a493457b-acb1-4029-bcc4-1d8290fa8759, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:55.677796039Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a493457b-acb1-4029-bcc4-1d8290fa8759, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:55.67785145Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:55.697139391Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:55.697199642Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:55.716227448Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:55.71628564Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1daa94e0-684d-4a00-9332-60a34c24e94a, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:55.735845167Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1daa94e0-684d-4a00-9332-60a34c24e94a, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:55.735897166Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:55.755049274Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:55.755111516Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:55.774301175Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:55.774350726Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fdc9e743-1329-4663-9bc9-0b2a07234456, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:55.793460193Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fdc9e743-1329-4663-9bc9-0b2a07234456, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:55.793511844Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:55.81192538Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:55.811986271Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:55.831411214Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:55.831477725Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:55.831677259Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:55.831818591Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:55.927351648Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:55.927427801Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:55.927443381Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:55.927450352Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:55.927458072Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:55.927465292Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:55.927472762Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:55.927478902Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:55.927484672Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:55.927498513Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:55.927523773Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:55.927528313Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:55.927534203Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:55.927538463Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:55.927549233Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:55.927554074Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:55.927561823Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:55.927567514Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:55.927573164Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:55.927623005Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"11fbd5db-03b3-4757-b600-85da5d37f518","remote_addr":"[fd00:1122:3344:101::1]:42546","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42\", TimeNotSynchronized), (\"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456\", TimeNotSynchronized), (\"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af\", TimeNotSynchronized), (\"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868\", TimeNotSynchronized), (\"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c\", TimeNotSynchronized), (\"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47\", TimeNotSynchronized), (\"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a\", TimeNotSynchronized), (\"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759\", TimeNotSynchronized), (\"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878\", TimeNotSynchronized), (\"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab\", TimeNotSynchronized), (\"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae\", TimeNotSynchronized), (\"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95\", TimeNotSynchronized), (\"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911\", TimeNotSynchronized), (\"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7\", TimeNotSynchronized), (\"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0\", TimeNotSynchronized), (\"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846\", TimeNotSynchronized), (\"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\", TimeNotSynchronized), (\"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff\", TimeNotSynchronized), (\"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2\", TimeNotSynchronized)]","latency_us":488708,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:55.927831858Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"644.28495ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"11fbd5db-03b3-4757-b600-85da5d37f518\", \"content-length\": \"124\", \"date\": \"Mon, 26 Feb 2024 07:12:55 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"11fbd5db-03b3-4757-b600-85da5d37f518\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: d0d03c23-79c4-4c60-898e-739f188b5615, kind: Service { id: 3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc }, name: Name(\"ntp-3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 196, 59])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 4b099cd4-61db-4587-84b8-cf9015577106, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 4c8e127e-7304-49fa-9de2-79bbde180d94, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: d60f3c38-1add-47dd-baca-a6103e9614ae, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 864c4b26-c35e-42dc-a5b0-3e041205ccbc, kind: Service { id: d60f3c38-1add-47dd-baca-a6103e9614ae }, name: Name(\"external-dns-d60f3c38-1add-47dd-baca-a6103e9614ae\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 7])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: c9ac5e72-089e-4e66-8c75-d1c69a21aee1, kind: Service { id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95 }, name: Name(\"external-dns-740e2473-798e-4bf1-9bf5-7a2c9ce38b95\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 248])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 60944a67-2d69-42d2-8ce7-1215cdc06eab, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 850653ea-9272-4e6c-b3ae-b169328ac41c, kind: Service { id: 60944a67-2d69-42d2-8ce7-1215cdc06eab }, name: Name(\"nexus-60944a67-2d69-42d2-8ce7-1215cdc06eab\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 217, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 3c85fee5-2c8b-46c2-a723-6c621ecfb3be, kind: Service { id: 3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7 }, name: Name(\"nexus-3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 215])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e72f50ee-9b20-4af9-becc-46f37972a3b0, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 03f00562-72e1-4d42-8124-11bb07005618, kind: Service { id: e72f50ee-9b20-4af9-becc-46f37972a3b0 }, name: Name(\"nexus-e72f50ee-9b20-4af9-becc-46f37972a3b0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 176, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7fcbadc5-146c-4a16-bbe8-9fe02e4fe911, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: fbaa903b-4740-4365-b35e-68123cc1d3a2, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1693d50c-fcf2-4249-86ae-6e12a08ac868, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 065d24ba-ce47-47d1-9175-26eabb479878, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: bc1764f3-5054-4735-919d-fb5def7e7e42, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a493457b-acb1-4029-bcc4-1d8290fa8759, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 1daa94e0-684d-4a00-9332-60a34c24e94a, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: fdc9e743-1329-4663-9bc9-0b2a07234456, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:56.571654357Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:56.572626442Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:56.572644625Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:56.593937861Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:56.593991782Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:56.615437451Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:56.615507179Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:56.634133351Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:56.634184882Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:56.652787121Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:56.652840242Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b099cd4-61db-4587-84b8-cf9015577106, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:56.670890711Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4b099cd4-61db-4587-84b8-cf9015577106, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:56.670941532Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:56.691527776Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:56.691581347Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4c8e127e-7304-49fa-9de2-79bbde180d94, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:56.710101554Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4c8e127e-7304-49fa-9de2-79bbde180d94, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:56.710146954Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d60f3c38-1add-47dd-baca-a6103e9614ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:56.729659829Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d60f3c38-1add-47dd-baca-a6103e9614ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:56.72971093Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:56.748971971Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:56.749034902Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:56.766177545Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:56.766227765Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bc1764f3-5054-4735-919d-fb5def7e7e42, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:56.785213761Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bc1764f3-5054-4735-919d-fb5def7e7e42, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:56.785279632Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a493457b-acb1-4029-bcc4-1d8290fa8759, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:56.804716556Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a493457b-acb1-4029-bcc4-1d8290fa8759, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:56.804768717Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:56.823506918Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:56.823558549Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:56.843234487Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:56.843287267Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1daa94e0-684d-4a00-9332-60a34c24e94a, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:56.862069939Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1daa94e0-684d-4a00-9332-60a34c24e94a, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:56.862132861Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:56.881552394Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:56.881618155Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:56.899808206Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:56.899858817Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fdc9e743-1329-4663-9bc9-0b2a07234456, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:56.918223592Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fdc9e743-1329-4663-9bc9-0b2a07234456, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:56.918279443Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:56.937158827Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:56.937215608Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:56.956658541Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:56.956732552Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:56.956956627Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:56.957093459Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:57.055586Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:57.055651681Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:57.055660281Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:57.055666811Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:57.055673521Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:57.055680271Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:57.055685972Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:57.055710582Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:57.055716712Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:57.055722252Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:57.055728292Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:57.055735442Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:57.055742933Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:57.055748482Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:57.055755833Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:57.055760203Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:57.055764373Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:57.055770433Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:57.055774563Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:57.055861024Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"1115858d-3c85-4cb5-ae02-1b7e069a5c31","remote_addr":"[fd00:1122:3344:101::1]:42546","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846\", TimeNotSynchronized), (\"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456\", TimeNotSynchronized), (\"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a\", TimeNotSynchronized), (\"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911\", TimeNotSynchronized), (\"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae\", TimeNotSynchronized), (\"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c\", TimeNotSynchronized), (\"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2\", TimeNotSynchronized), (\"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af\", TimeNotSynchronized), (\"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\", TimeNotSynchronized), (\"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0\", TimeNotSynchronized), (\"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff\", TimeNotSynchronized), (\"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868\", TimeNotSynchronized), (\"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95\", TimeNotSynchronized), (\"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7\", TimeNotSynchronized), (\"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42\", TimeNotSynchronized), (\"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47\", TimeNotSynchronized), (\"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab\", TimeNotSynchronized), (\"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878\", TimeNotSynchronized), (\"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759\", TimeNotSynchronized)]","latency_us":484774,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:57.056099459Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"415.49719ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1115858d-3c85-4cb5-ae02-1b7e069a5c31\", \"content-length\": \"124\", \"date\": \"Mon, 26 Feb 2024 07:12:56 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"1115858d-3c85-4cb5-ae02-1b7e069a5c31\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: d0d03c23-79c4-4c60-898e-739f188b5615, kind: Service { id: 3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc }, name: Name(\"ntp-3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 196, 59])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 4b099cd4-61db-4587-84b8-cf9015577106, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 4c8e127e-7304-49fa-9de2-79bbde180d94, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: d60f3c38-1add-47dd-baca-a6103e9614ae, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 864c4b26-c35e-42dc-a5b0-3e041205ccbc, kind: Service { id: d60f3c38-1add-47dd-baca-a6103e9614ae }, name: Name(\"external-dns-d60f3c38-1add-47dd-baca-a6103e9614ae\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 7])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: c9ac5e72-089e-4e66-8c75-d1c69a21aee1, kind: Service { id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95 }, name: Name(\"external-dns-740e2473-798e-4bf1-9bf5-7a2c9ce38b95\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 248])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 60944a67-2d69-42d2-8ce7-1215cdc06eab, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 850653ea-9272-4e6c-b3ae-b169328ac41c, kind: Service { id: 60944a67-2d69-42d2-8ce7-1215cdc06eab }, name: Name(\"nexus-60944a67-2d69-42d2-8ce7-1215cdc06eab\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 217, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 3c85fee5-2c8b-46c2-a723-6c621ecfb3be, kind: Service { id: 3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7 }, name: Name(\"nexus-3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 215])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e72f50ee-9b20-4af9-becc-46f37972a3b0, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 03f00562-72e1-4d42-8124-11bb07005618, kind: Service { id: e72f50ee-9b20-4af9-becc-46f37972a3b0 }, name: Name(\"nexus-e72f50ee-9b20-4af9-becc-46f37972a3b0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 176, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7fcbadc5-146c-4a16-bbe8-9fe02e4fe911, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: fbaa903b-4740-4365-b35e-68123cc1d3a2, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1693d50c-fcf2-4249-86ae-6e12a08ac868, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 065d24ba-ce47-47d1-9175-26eabb479878, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: bc1764f3-5054-4735-919d-fb5def7e7e42, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a493457b-acb1-4029-bcc4-1d8290fa8759, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 1daa94e0-684d-4a00-9332-60a34c24e94a, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: fdc9e743-1329-4663-9bc9-0b2a07234456, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:57.471426499Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:57.472075941Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:57.472096311Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:57.494289963Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:57.494352774Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:57.513641405Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:57.513693126Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:57.53203874Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:57.532102752Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:57.550623079Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:57.55068309Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b099cd4-61db-4587-84b8-cf9015577106, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:57.569897089Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4b099cd4-61db-4587-84b8-cf9015577106, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:57.569948181Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:57.585494135Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:57.585547906Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4c8e127e-7304-49fa-9de2-79bbde180d94, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:57.606935294Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4c8e127e-7304-49fa-9de2-79bbde180d94, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:57.609400868Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d60f3c38-1add-47dd-baca-a6103e9614ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:57.625693866Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d60f3c38-1add-47dd-baca-a6103e9614ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:57.625748831Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:57.644290744Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:57.644344245Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:57.665350487Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:57.665411648Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bc1764f3-5054-4735-919d-fb5def7e7e42, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:57.686635333Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bc1764f3-5054-4735-919d-fb5def7e7e42, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:57.686695024Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a493457b-acb1-4029-bcc4-1d8290fa8759, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:57.705924423Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a493457b-acb1-4029-bcc4-1d8290fa8759, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:57.705983285Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:57.72612395Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:57.726184611Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:57.746236386Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:57.746303967Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1daa94e0-684d-4a00-9332-60a34c24e94a, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:57.767514352Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1daa94e0-684d-4a00-9332-60a34c24e94a, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:57.767578873Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:57.787012166Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:57.787079098Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:57.807196413Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:57.807257874Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fdc9e743-1329-4663-9bc9-0b2a07234456, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:57.826618706Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fdc9e743-1329-4663-9bc9-0b2a07234456, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:57.826684758Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:57.846362846Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:57.846421856Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:57.866229557Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:57.866301818Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:57.866521392Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:57.866672934Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:57.961277237Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:57.961360258Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:57.961367668Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:57.961374938Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:57.961399159Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:57.961406969Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:57.961420479Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:57.961428299Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:57.961435909Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:57.961443429Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:57.961450949Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:57.96145596Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:57.96146223Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:57.96147165Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:57.96147767Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:57.96148214Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:57.96148826Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:57.96149544Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:57.961500861Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:57.961553791Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"13f9a84d-d284-4ca7-a2c3-78ac077b93c3","remote_addr":"[fd00:1122:3344:101::1]:42546","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42\", TimeNotSynchronized), (\"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47\", TimeNotSynchronized), (\"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae\", TimeNotSynchronized), (\"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab\", TimeNotSynchronized), (\"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7\", TimeNotSynchronized), (\"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2\", TimeNotSynchronized), (\"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\", TimeNotSynchronized), (\"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c\", TimeNotSynchronized), (\"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a\", TimeNotSynchronized), (\"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456\", TimeNotSynchronized), (\"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846\", TimeNotSynchronized), (\"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878\", TimeNotSynchronized), (\"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af\", TimeNotSynchronized), (\"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95\", TimeNotSynchronized), (\"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911\", TimeNotSynchronized), (\"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868\", TimeNotSynchronized), (\"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759\", TimeNotSynchronized), (\"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff\", TimeNotSynchronized), (\"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0\", TimeNotSynchronized)]","latency_us":491015,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:57.961774615Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"585.983824ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"13f9a84d-d284-4ca7-a2c3-78ac077b93c3\", \"content-length\": \"124\", \"date\": \"Mon, 26 Feb 2024 07:12:57 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"13f9a84d-d284-4ca7-a2c3-78ac077b93c3\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: d0d03c23-79c4-4c60-898e-739f188b5615, kind: Service { id: 3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc }, name: Name(\"ntp-3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 196, 59])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 4b099cd4-61db-4587-84b8-cf9015577106, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 4c8e127e-7304-49fa-9de2-79bbde180d94, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: d60f3c38-1add-47dd-baca-a6103e9614ae, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 864c4b26-c35e-42dc-a5b0-3e041205ccbc, kind: Service { id: d60f3c38-1add-47dd-baca-a6103e9614ae }, name: Name(\"external-dns-d60f3c38-1add-47dd-baca-a6103e9614ae\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 7])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: c9ac5e72-089e-4e66-8c75-d1c69a21aee1, kind: Service { id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95 }, name: Name(\"external-dns-740e2473-798e-4bf1-9bf5-7a2c9ce38b95\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 248])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 60944a67-2d69-42d2-8ce7-1215cdc06eab, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 850653ea-9272-4e6c-b3ae-b169328ac41c, kind: Service { id: 60944a67-2d69-42d2-8ce7-1215cdc06eab }, name: Name(\"nexus-60944a67-2d69-42d2-8ce7-1215cdc06eab\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 217, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 3c85fee5-2c8b-46c2-a723-6c621ecfb3be, kind: Service { id: 3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7 }, name: Name(\"nexus-3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 215])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e72f50ee-9b20-4af9-becc-46f37972a3b0, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 03f00562-72e1-4d42-8124-11bb07005618, kind: Service { id: e72f50ee-9b20-4af9-becc-46f37972a3b0 }, name: Name(\"nexus-e72f50ee-9b20-4af9-becc-46f37972a3b0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 176, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7fcbadc5-146c-4a16-bbe8-9fe02e4fe911, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: fbaa903b-4740-4365-b35e-68123cc1d3a2, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1693d50c-fcf2-4249-86ae-6e12a08ac868, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 065d24ba-ce47-47d1-9175-26eabb479878, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: bc1764f3-5054-4735-919d-fb5def7e7e42, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a493457b-acb1-4029-bcc4-1d8290fa8759, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 1daa94e0-684d-4a00-9332-60a34c24e94a, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: fdc9e743-1329-4663-9bc9-0b2a07234456, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:58.546716753Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:58.547358795Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:58.547375435Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:58.570712377Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:58.570775129Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:58.590403256Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:58.590460787Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:58.610031183Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:58.610088577Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:58.626662266Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:58.626717907Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b099cd4-61db-4587-84b8-cf9015577106, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:58.645280915Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4b099cd4-61db-4587-84b8-cf9015577106, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:58.645337836Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:58.66422304Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:58.664297822Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4c8e127e-7304-49fa-9de2-79bbde180d94, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:58.686409282Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4c8e127e-7304-49fa-9de2-79bbde180d94, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:58.686476414Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d60f3c38-1add-47dd-baca-a6103e9614ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:58.706622459Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d60f3c38-1add-47dd-baca-a6103e9614ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:58.70667878Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:58.727663331Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:58.727720182Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:58.747590534Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:58.747657705Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bc1764f3-5054-4735-919d-fb5def7e7e42, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:58.76834689Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bc1764f3-5054-4735-919d-fb5def7e7e42, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:58.768412151Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a493457b-acb1-4029-bcc4-1d8290fa8759, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:58.78811641Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a493457b-acb1-4029-bcc4-1d8290fa8759, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:58.788176591Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:58.807450312Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:58.807509442Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:58.826412237Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:58.826477548Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1daa94e0-684d-4a00-9332-60a34c24e94a, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:58.84696721Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1daa94e0-684d-4a00-9332-60a34c24e94a, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:58.847029211Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:58.866717059Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:58.86677764Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:58.8837561Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:58.883816141Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fdc9e743-1329-4663-9bc9-0b2a07234456, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:58.903815605Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fdc9e743-1329-4663-9bc9-0b2a07234456, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:58.903877716Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:58.924580062Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:58.924639673Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:58.94488499Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:58.944956842Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:58.945163986Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:58.945301138Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:59.042284652Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:59.042381944Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:59.042391184Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:59.042398914Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:59.042405584Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:59.042412454Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:59.042418644Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:59.042425545Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:59.042436175Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:59.042442895Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:59.042447305Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:59.042451775Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:59.042457205Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:59.042462845Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:59.042468575Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:59.042474255Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:59.042481846Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:59.042491326Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:59.042502656Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:12:59.042536496Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"99b964fc-5dfd-4aef-a5f3-4bb7932ab707","remote_addr":"[fd00:1122:3344:101::1]:42546","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab\", TimeNotSynchronized), (\"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2\", TimeNotSynchronized), (\"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c\", TimeNotSynchronized), (\"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47\", TimeNotSynchronized), (\"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42\", TimeNotSynchronized), (\"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911\", TimeNotSynchronized), (\"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a\", TimeNotSynchronized), (\"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456\", TimeNotSynchronized), (\"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868\", TimeNotSynchronized), (\"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846\", TimeNotSynchronized), (\"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff\", TimeNotSynchronized), (\"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af\", TimeNotSynchronized), (\"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95\", TimeNotSynchronized), (\"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0\", TimeNotSynchronized), (\"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7\", TimeNotSynchronized), (\"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae\", TimeNotSynchronized), (\"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878\", TimeNotSynchronized), (\"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\", TimeNotSynchronized), (\"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759\", TimeNotSynchronized)]","latency_us":496715,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:12:59.04275366Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"1.067770198s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"99b964fc-5dfd-4aef-a5f3-4bb7932ab707\", \"content-length\": \"124\", \"date\": \"Mon, 26 Feb 2024 07:12:58 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"99b964fc-5dfd-4aef-a5f3-4bb7932ab707\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: d0d03c23-79c4-4c60-898e-739f188b5615, kind: Service { id: 3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc }, name: Name(\"ntp-3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 196, 59])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 4b099cd4-61db-4587-84b8-cf9015577106, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 4c8e127e-7304-49fa-9de2-79bbde180d94, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: d60f3c38-1add-47dd-baca-a6103e9614ae, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 864c4b26-c35e-42dc-a5b0-3e041205ccbc, kind: Service { id: d60f3c38-1add-47dd-baca-a6103e9614ae }, name: Name(\"external-dns-d60f3c38-1add-47dd-baca-a6103e9614ae\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 7])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: c9ac5e72-089e-4e66-8c75-d1c69a21aee1, kind: Service { id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95 }, name: Name(\"external-dns-740e2473-798e-4bf1-9bf5-7a2c9ce38b95\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 248])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 60944a67-2d69-42d2-8ce7-1215cdc06eab, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 850653ea-9272-4e6c-b3ae-b169328ac41c, kind: Service { id: 60944a67-2d69-42d2-8ce7-1215cdc06eab }, name: Name(\"nexus-60944a67-2d69-42d2-8ce7-1215cdc06eab\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 217, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 3c85fee5-2c8b-46c2-a723-6c621ecfb3be, kind: Service { id: 3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7 }, name: Name(\"nexus-3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 215])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e72f50ee-9b20-4af9-becc-46f37972a3b0, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 03f00562-72e1-4d42-8124-11bb07005618, kind: Service { id: e72f50ee-9b20-4af9-becc-46f37972a3b0 }, name: Name(\"nexus-e72f50ee-9b20-4af9-becc-46f37972a3b0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 176, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7fcbadc5-146c-4a16-bbe8-9fe02e4fe911, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: fbaa903b-4740-4365-b35e-68123cc1d3a2, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1693d50c-fcf2-4249-86ae-6e12a08ac868, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 065d24ba-ce47-47d1-9175-26eabb479878, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: bc1764f3-5054-4735-919d-fb5def7e7e42, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a493457b-acb1-4029-bcc4-1d8290fa8759, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 1daa94e0-684d-4a00-9332-60a34c24e94a, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: fdc9e743-1329-4663-9bc9-0b2a07234456, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:00.108275842Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:00.10905445Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:00.109079306Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:00.132822216Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:00.132885287Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:00.153396499Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:00.155911321Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:00.173044116Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:00.173112258Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:00.192581002Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:00.192639943Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b099cd4-61db-4587-84b8-cf9015577106, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:00.210517449Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4b099cd4-61db-4587-84b8-cf9015577106, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:00.21057696Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:00.231582471Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:00.231646392Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4c8e127e-7304-49fa-9de2-79bbde180d94, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:00.250423194Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4c8e127e-7304-49fa-9de2-79bbde180d94, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:00.250491185Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d60f3c38-1add-47dd-baca-a6103e9614ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:00.269397159Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d60f3c38-1add-47dd-baca-a6103e9614ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:00.26945678Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:00.289513175Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:00.2895705Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:00.309185602Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:00.309246424Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bc1764f3-5054-4735-919d-fb5def7e7e42, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:00.329590653Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bc1764f3-5054-4735-919d-fb5def7e7e42, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:00.329652984Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a493457b-acb1-4029-bcc4-1d8290fa8759, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:00.349140089Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a493457b-acb1-4029-bcc4-1d8290fa8759, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:00.3491947Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:00.368001522Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:00.368073133Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:00.386665052Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:00.386721533Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1daa94e0-684d-4a00-9332-60a34c24e94a, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:00.405548356Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1daa94e0-684d-4a00-9332-60a34c24e94a, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:00.405599007Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:00.425222153Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:00.425278805Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:00.445891069Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:00.44594787Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fdc9e743-1329-4663-9bc9-0b2a07234456, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:00.464774483Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fdc9e743-1329-4663-9bc9-0b2a07234456, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:00.464834624Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:00.48305493Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:00.483113067Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:00.502786474Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:00.502872086Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:00.503056769Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:00.503197782Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:00.6007461Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:00.600833727Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:00.600844457Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:00.600854457Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:00.600861448Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:00.600869948Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:00.600878568Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:00.600889158Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:00.600911849Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:00.600920649Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:00.600929099Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:00.600937659Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:00.600945909Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:00.600951259Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:00.60096571Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:00.60097119Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:00.60097796Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:00.60099204Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:00.60099915Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:00.601064071Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"c0b07346-8dc0-4faf-b7ef-2def79f61869","remote_addr":"[fd00:1122:3344:101::1]:42546","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911\", TimeNotSynchronized), (\"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae\", TimeNotSynchronized), (\"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42\", TimeNotSynchronized), (\"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759\", TimeNotSynchronized), (\"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95\", TimeNotSynchronized), (\"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868\", TimeNotSynchronized), (\"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846\", TimeNotSynchronized), (\"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2\", TimeNotSynchronized), (\"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47\", TimeNotSynchronized), (\"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\", TimeNotSynchronized), (\"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878\", TimeNotSynchronized), (\"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c\", TimeNotSynchronized), (\"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af\", TimeNotSynchronized), (\"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab\", TimeNotSynchronized), (\"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7\", TimeNotSynchronized), (\"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff\", TimeNotSynchronized), (\"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0\", TimeNotSynchronized), (\"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a\", TimeNotSynchronized), (\"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456\", TimeNotSynchronized)]","latency_us":493568,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:00.601284085Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"754.988484ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c0b07346-8dc0-4faf-b7ef-2def79f61869\", \"content-length\": \"124\", \"date\": \"Mon, 26 Feb 2024 07:13:00 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c0b07346-8dc0-4faf-b7ef-2def79f61869\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: d0d03c23-79c4-4c60-898e-739f188b5615, kind: Service { id: 3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc }, name: Name(\"ntp-3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 196, 59])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 4b099cd4-61db-4587-84b8-cf9015577106, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 4c8e127e-7304-49fa-9de2-79bbde180d94, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: d60f3c38-1add-47dd-baca-a6103e9614ae, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 864c4b26-c35e-42dc-a5b0-3e041205ccbc, kind: Service { id: d60f3c38-1add-47dd-baca-a6103e9614ae }, name: Name(\"external-dns-d60f3c38-1add-47dd-baca-a6103e9614ae\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 7])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: c9ac5e72-089e-4e66-8c75-d1c69a21aee1, kind: Service { id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95 }, name: Name(\"external-dns-740e2473-798e-4bf1-9bf5-7a2c9ce38b95\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 248])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 60944a67-2d69-42d2-8ce7-1215cdc06eab, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 850653ea-9272-4e6c-b3ae-b169328ac41c, kind: Service { id: 60944a67-2d69-42d2-8ce7-1215cdc06eab }, name: Name(\"nexus-60944a67-2d69-42d2-8ce7-1215cdc06eab\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 217, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 3c85fee5-2c8b-46c2-a723-6c621ecfb3be, kind: Service { id: 3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7 }, name: Name(\"nexus-3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 215])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e72f50ee-9b20-4af9-becc-46f37972a3b0, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 03f00562-72e1-4d42-8124-11bb07005618, kind: Service { id: e72f50ee-9b20-4af9-becc-46f37972a3b0 }, name: Name(\"nexus-e72f50ee-9b20-4af9-becc-46f37972a3b0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 176, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7fcbadc5-146c-4a16-bbe8-9fe02e4fe911, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: fbaa903b-4740-4365-b35e-68123cc1d3a2, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1693d50c-fcf2-4249-86ae-6e12a08ac868, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 065d24ba-ce47-47d1-9175-26eabb479878, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: bc1764f3-5054-4735-919d-fb5def7e7e42, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a493457b-acb1-4029-bcc4-1d8290fa8759, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 1daa94e0-684d-4a00-9332-60a34c24e94a, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: fdc9e743-1329-4663-9bc9-0b2a07234456, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:01.355556486Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:01.356304019Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:01.35631956Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:01.379776084Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:01.379849275Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:01.401416737Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:01.401488898Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:01.422167843Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:01.422231064Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:01.441870592Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:01.441929753Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b099cd4-61db-4587-84b8-cf9015577106, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:01.462482816Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4b099cd4-61db-4587-84b8-cf9015577106, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:01.464830687Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:01.482838316Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:01.482899017Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4c8e127e-7304-49fa-9de2-79bbde180d94, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:01.50232841Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4c8e127e-7304-49fa-9de2-79bbde180d94, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:01.502392411Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d60f3c38-1add-47dd-baca-a6103e9614ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:01.521843455Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d60f3c38-1add-47dd-baca-a6103e9614ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:01.521906326Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:01.540629897Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:01.540696678Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:01.561366693Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:01.561424405Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bc1764f3-5054-4735-919d-fb5def7e7e42, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:01.58263962Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bc1764f3-5054-4735-919d-fb5def7e7e42, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:01.58269564Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a493457b-acb1-4029-bcc4-1d8290fa8759, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:01.602097653Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a493457b-acb1-4029-bcc4-1d8290fa8759, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:01.602155734Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:01.621401474Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:01.621470606Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:01.641379427Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:01.641432339Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1daa94e0-684d-4a00-9332-60a34c24e94a, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:01.661448442Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1daa94e0-684d-4a00-9332-60a34c24e94a, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:01.661506933Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:01.681760351Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:01.681827222Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:01.70434692Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:01.704407521Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fdc9e743-1329-4663-9bc9-0b2a07234456, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:01.723904816Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fdc9e743-1329-4663-9bc9-0b2a07234456, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:01.723960787Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:01.744685973Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:01.744741534Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:01.765012012Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:01.765082414Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:01.765273197Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:01.765409869Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:01.863435852Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:01.863508473Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:01.863518513Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:01.863532864Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:01.863538854Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:01.863545774Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:01.863551294Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:01.863558584Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:01.863564504Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:01.863571904Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:01.863577514Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:01.863595675Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:01.863601675Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:01.863606135Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:01.863616735Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:01.863621215Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:01.863626645Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:01.863632865Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:01.863638675Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:01.863703547Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"1c62a946-8b98-44de-a3ed-bdc432c13159","remote_addr":"[fd00:1122:3344:101::1]:42546","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae\", TimeNotSynchronized), (\"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846\", TimeNotSynchronized), (\"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2\", TimeNotSynchronized), (\"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab\", TimeNotSynchronized), (\"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95\", TimeNotSynchronized), (\"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0\", TimeNotSynchronized), (\"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868\", TimeNotSynchronized), (\"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878\", TimeNotSynchronized), (\"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47\", TimeNotSynchronized), (\"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff\", TimeNotSynchronized), (\"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456\", TimeNotSynchronized), (\"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c\", TimeNotSynchronized), (\"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\", TimeNotSynchronized), (\"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a\", TimeNotSynchronized), (\"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af\", TimeNotSynchronized), (\"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911\", TimeNotSynchronized), (\"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7\", TimeNotSynchronized), (\"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42\", TimeNotSynchronized), (\"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759\", TimeNotSynchronized)]","latency_us":509003,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:01.863925871Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"1.478827833s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1c62a946-8b98-44de-a3ed-bdc432c13159\", \"content-length\": \"124\", \"date\": \"Mon, 26 Feb 2024 07:13:01 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"1c62a946-8b98-44de-a3ed-bdc432c13159\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: d0d03c23-79c4-4c60-898e-739f188b5615, kind: Service { id: 3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc }, name: Name(\"ntp-3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 196, 59])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 4b099cd4-61db-4587-84b8-cf9015577106, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 4c8e127e-7304-49fa-9de2-79bbde180d94, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: d60f3c38-1add-47dd-baca-a6103e9614ae, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 864c4b26-c35e-42dc-a5b0-3e041205ccbc, kind: Service { id: d60f3c38-1add-47dd-baca-a6103e9614ae }, name: Name(\"external-dns-d60f3c38-1add-47dd-baca-a6103e9614ae\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 7])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: c9ac5e72-089e-4e66-8c75-d1c69a21aee1, kind: Service { id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95 }, name: Name(\"external-dns-740e2473-798e-4bf1-9bf5-7a2c9ce38b95\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 248])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 60944a67-2d69-42d2-8ce7-1215cdc06eab, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 850653ea-9272-4e6c-b3ae-b169328ac41c, kind: Service { id: 60944a67-2d69-42d2-8ce7-1215cdc06eab }, name: Name(\"nexus-60944a67-2d69-42d2-8ce7-1215cdc06eab\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 217, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 3c85fee5-2c8b-46c2-a723-6c621ecfb3be, kind: Service { id: 3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7 }, name: Name(\"nexus-3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 215])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e72f50ee-9b20-4af9-becc-46f37972a3b0, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 03f00562-72e1-4d42-8124-11bb07005618, kind: Service { id: e72f50ee-9b20-4af9-becc-46f37972a3b0 }, name: Name(\"nexus-e72f50ee-9b20-4af9-becc-46f37972a3b0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 176, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7fcbadc5-146c-4a16-bbe8-9fe02e4fe911, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: fbaa903b-4740-4365-b35e-68123cc1d3a2, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1693d50c-fcf2-4249-86ae-6e12a08ac868, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 065d24ba-ce47-47d1-9175-26eabb479878, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: bc1764f3-5054-4735-919d-fb5def7e7e42, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a493457b-acb1-4029-bcc4-1d8290fa8759, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 1daa94e0-684d-4a00-9332-60a34c24e94a, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: fdc9e743-1329-4663-9bc9-0b2a07234456, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:03.340249083Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:03.340890634Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:03.340911214Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:03.363690227Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:03.363749268Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:03.384207937Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:03.384265681Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:03.405914233Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:03.405971104Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:03.425033761Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:03.427325942Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b099cd4-61db-4587-84b8-cf9015577106, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:03.442057182Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4b099cd4-61db-4587-84b8-cf9015577106, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:03.442115373Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:03.462771658Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:03.462838319Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4c8e127e-7304-49fa-9de2-79bbde180d94, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:03.482045729Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4c8e127e-7304-49fa-9de2-79bbde180d94, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:03.48210543Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d60f3c38-1add-47dd-baca-a6103e9614ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:03.501869391Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d60f3c38-1add-47dd-baca-a6103e9614ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:03.5019339Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:03.521531057Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:03.521587708Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:03.542298304Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:03.542355715Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bc1764f3-5054-4735-919d-fb5def7e7e42, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:03.560893252Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bc1764f3-5054-4735-919d-fb5def7e7e42, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:03.560955853Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a493457b-acb1-4029-bcc4-1d8290fa8759, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:03.580324486Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a493457b-acb1-4029-bcc4-1d8290fa8759, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:03.580380097Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:03.601530481Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:03.601592542Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:03.620477915Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:03.620532666Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1daa94e0-684d-4a00-9332-60a34c24e94a, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:03.640671532Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1daa94e0-684d-4a00-9332-60a34c24e94a, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:03.640743614Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:03.663675709Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:03.6637335Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:03.683837065Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:03.683903386Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fdc9e743-1329-4663-9bc9-0b2a07234456, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:03.703984501Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fdc9e743-1329-4663-9bc9-0b2a07234456, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:03.704039492Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:03.724695517Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:03.724779408Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:03.744255083Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:03.744326854Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:03.744509277Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:03.74464683Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:03.842446478Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:03.84253214Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:03.84254621Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:03.84255266Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:03.84255966Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:03.84256545Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:03.84257266Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:03.84258032Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:03.842594871Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:03.842600941Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:03.842607871Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:03.842614871Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:03.842629471Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:03.842634751Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:03.842640502Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:03.842645062Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:03.842649531Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:03.842655532Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:03.842661292Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:03.842727063Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"f39dd281-b2bf-4269-ad6d-10a37ca4f255","remote_addr":"[fd00:1122:3344:101::1]:42546","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47\", TimeNotSynchronized), (\"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868\", TimeNotSynchronized), (\"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846\", TimeNotSynchronized), (\"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\", TimeNotSynchronized), (\"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911\", TimeNotSynchronized), (\"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c\", TimeNotSynchronized), (\"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95\", TimeNotSynchronized), (\"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42\", TimeNotSynchronized), (\"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae\", TimeNotSynchronized), (\"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759\", TimeNotSynchronized), (\"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a\", TimeNotSynchronized), (\"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff\", TimeNotSynchronized), (\"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7\", TimeNotSynchronized), (\"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2\", TimeNotSynchronized), (\"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0\", TimeNotSynchronized), (\"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab\", TimeNotSynchronized), (\"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456\", TimeNotSynchronized), (\"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878\", TimeNotSynchronized), (\"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af\", TimeNotSynchronized)]","latency_us":503393,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:03.842946529Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"2.117638243s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f39dd281-b2bf-4269-ad6d-10a37ca4f255\", \"content-length\": \"124\", \"date\": \"Mon, 26 Feb 2024 07:13:03 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f39dd281-b2bf-4269-ad6d-10a37ca4f255\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: d0d03c23-79c4-4c60-898e-739f188b5615, kind: Service { id: 3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc }, name: Name(\"ntp-3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 196, 59])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 4b099cd4-61db-4587-84b8-cf9015577106, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 4c8e127e-7304-49fa-9de2-79bbde180d94, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: d60f3c38-1add-47dd-baca-a6103e9614ae, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 864c4b26-c35e-42dc-a5b0-3e041205ccbc, kind: Service { id: d60f3c38-1add-47dd-baca-a6103e9614ae }, name: Name(\"external-dns-d60f3c38-1add-47dd-baca-a6103e9614ae\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 7])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: c9ac5e72-089e-4e66-8c75-d1c69a21aee1, kind: Service { id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95 }, name: Name(\"external-dns-740e2473-798e-4bf1-9bf5-7a2c9ce38b95\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 248])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 60944a67-2d69-42d2-8ce7-1215cdc06eab, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 850653ea-9272-4e6c-b3ae-b169328ac41c, kind: Service { id: 60944a67-2d69-42d2-8ce7-1215cdc06eab }, name: Name(\"nexus-60944a67-2d69-42d2-8ce7-1215cdc06eab\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 217, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 3c85fee5-2c8b-46c2-a723-6c621ecfb3be, kind: Service { id: 3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7 }, name: Name(\"nexus-3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 215])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e72f50ee-9b20-4af9-becc-46f37972a3b0, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 03f00562-72e1-4d42-8124-11bb07005618, kind: Service { id: e72f50ee-9b20-4af9-becc-46f37972a3b0 }, name: Name(\"nexus-e72f50ee-9b20-4af9-becc-46f37972a3b0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 176, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7fcbadc5-146c-4a16-bbe8-9fe02e4fe911, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: fbaa903b-4740-4365-b35e-68123cc1d3a2, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1693d50c-fcf2-4249-86ae-6e12a08ac868, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 065d24ba-ce47-47d1-9175-26eabb479878, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: bc1764f3-5054-4735-919d-fb5def7e7e42, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a493457b-acb1-4029-bcc4-1d8290fa8759, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 1daa94e0-684d-4a00-9332-60a34c24e94a, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: fdc9e743-1329-4663-9bc9-0b2a07234456, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:05.955986932Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:05.956652194Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:05.956671454Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:05.980567026Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:05.980629988Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:06.002719988Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:06.002780419Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:06.024941651Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:06.025006552Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:06.046349869Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:06.0464114Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b099cd4-61db-4587-84b8-cf9015577106, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:06.067126316Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4b099cd4-61db-4587-84b8-cf9015577106, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:06.067188297Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:06.087525657Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:06.087585728Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4c8e127e-7304-49fa-9de2-79bbde180d94, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:06.106421711Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4c8e127e-7304-49fa-9de2-79bbde180d94, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:06.106486172Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d60f3c38-1add-47dd-baca-a6103e9614ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:06.126795381Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d60f3c38-1add-47dd-baca-a6103e9614ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:06.126858402Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:06.147176561Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:06.147233232Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:06.167127834Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:06.167185855Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bc1764f3-5054-4735-919d-fb5def7e7e42, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:06.188377189Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bc1764f3-5054-4735-919d-fb5def7e7e42, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:06.18845167Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a493457b-acb1-4029-bcc4-1d8290fa8759, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:06.207142861Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a493457b-acb1-4029-bcc4-1d8290fa8759, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:06.207210262Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:06.227129394Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:06.227194255Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:06.246752471Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:06.246821522Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1daa94e0-684d-4a00-9332-60a34c24e94a, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:06.267286244Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1daa94e0-684d-4a00-9332-60a34c24e94a, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:06.267349205Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:06.28633253Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:06.286390221Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:06.306780152Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:06.306843743Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fdc9e743-1329-4663-9bc9-0b2a07234456, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:06.327084621Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fdc9e743-1329-4663-9bc9-0b2a07234456, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:06.327147162Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:06.347198826Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:06.347255807Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:06.367614587Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:06.367685098Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:06.367877962Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:06.368016734Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:06.467652765Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:06.467731857Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:06.467741086Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:06.467748017Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:06.467755397Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:06.467773667Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:06.467779487Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:06.467783847Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:06.467794767Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:06.467808048Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:06.467814308Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:06.467819878Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:06.467825598Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:06.467833878Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:06.467838148Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:06.467842718Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:06.467846959Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:06.467852647Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:06.467856556Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:06.467877378Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"f7b0682b-f3a1-43ed-bb71-936afdf561b7","remote_addr":"[fd00:1122:3344:101::1]:42546","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff\", TimeNotSynchronized), (\"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456\", TimeNotSynchronized), (\"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846\", TimeNotSynchronized), (\"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab\", TimeNotSynchronized), (\"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c\", TimeNotSynchronized), (\"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868\", TimeNotSynchronized), (\"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95\", TimeNotSynchronized), (\"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47\", TimeNotSynchronized), (\"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a\", TimeNotSynchronized), (\"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42\", TimeNotSynchronized), (\"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759\", TimeNotSynchronized), (\"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7\", TimeNotSynchronized), (\"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af\", TimeNotSynchronized), (\"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\", TimeNotSynchronized), (\"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911\", TimeNotSynchronized), (\"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae\", TimeNotSynchronized), (\"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0\", TimeNotSynchronized), (\"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878\", TimeNotSynchronized), (\"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2\", TimeNotSynchronized)]","latency_us":512811,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:06.468103603Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"983.143564ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f7b0682b-f3a1-43ed-bb71-936afdf561b7\", \"content-length\": \"124\", \"date\": \"Mon, 26 Feb 2024 07:13:05 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f7b0682b-f3a1-43ed-bb71-936afdf561b7\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: d0d03c23-79c4-4c60-898e-739f188b5615, kind: Service { id: 3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc }, name: Name(\"ntp-3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 196, 59])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 4b099cd4-61db-4587-84b8-cf9015577106, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 4c8e127e-7304-49fa-9de2-79bbde180d94, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: d60f3c38-1add-47dd-baca-a6103e9614ae, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 864c4b26-c35e-42dc-a5b0-3e041205ccbc, kind: Service { id: d60f3c38-1add-47dd-baca-a6103e9614ae }, name: Name(\"external-dns-d60f3c38-1add-47dd-baca-a6103e9614ae\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 7])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: c9ac5e72-089e-4e66-8c75-d1c69a21aee1, kind: Service { id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95 }, name: Name(\"external-dns-740e2473-798e-4bf1-9bf5-7a2c9ce38b95\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 248])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 60944a67-2d69-42d2-8ce7-1215cdc06eab, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 850653ea-9272-4e6c-b3ae-b169328ac41c, kind: Service { id: 60944a67-2d69-42d2-8ce7-1215cdc06eab }, name: Name(\"nexus-60944a67-2d69-42d2-8ce7-1215cdc06eab\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 217, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 3c85fee5-2c8b-46c2-a723-6c621ecfb3be, kind: Service { id: 3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7 }, name: Name(\"nexus-3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 215])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e72f50ee-9b20-4af9-becc-46f37972a3b0, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 03f00562-72e1-4d42-8124-11bb07005618, kind: Service { id: e72f50ee-9b20-4af9-becc-46f37972a3b0 }, name: Name(\"nexus-e72f50ee-9b20-4af9-becc-46f37972a3b0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 176, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7fcbadc5-146c-4a16-bbe8-9fe02e4fe911, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: fbaa903b-4740-4365-b35e-68123cc1d3a2, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1693d50c-fcf2-4249-86ae-6e12a08ac868, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 065d24ba-ce47-47d1-9175-26eabb479878, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: bc1764f3-5054-4735-919d-fb5def7e7e42, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a493457b-acb1-4029-bcc4-1d8290fa8759, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 1daa94e0-684d-4a00-9332-60a34c24e94a, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: fdc9e743-1329-4663-9bc9-0b2a07234456, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:07.449858954Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:07.451022215Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:07.451040755Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:07.474184644Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:07.474241215Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:07.494668926Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:07.494716397Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:07.516164906Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:07.516239928Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:07.536048328Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:07.536102699Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b099cd4-61db-4587-84b8-cf9015577106, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:07.556138603Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4b099cd4-61db-4587-84b8-cf9015577106, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:07.556191874Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:07.575443014Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:07.575493535Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4c8e127e-7304-49fa-9de2-79bbde180d94, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:07.594882118Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4c8e127e-7304-49fa-9de2-79bbde180d94, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:07.594932299Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d60f3c38-1add-47dd-baca-a6103e9614ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:07.613801174Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d60f3c38-1add-47dd-baca-a6103e9614ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:07.613850663Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:07.633339107Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:07.633386878Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:07.653664469Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:07.653726967Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bc1764f3-5054-4735-919d-fb5def7e7e42, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:07.673791732Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bc1764f3-5054-4735-919d-fb5def7e7e42, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:07.673845103Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a493457b-acb1-4029-bcc4-1d8290fa8759, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:07.694019189Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a493457b-acb1-4029-bcc4-1d8290fa8759, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:07.69407912Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:07.714153655Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:07.714215546Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:07.734077057Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:07.736349518Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1daa94e0-684d-4a00-9332-60a34c24e94a, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:07.753634763Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1daa94e0-684d-4a00-9332-60a34c24e94a, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:07.753692774Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:07.772432595Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:07.772493606Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:07.791998171Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:07.792052452Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fdc9e743-1329-4663-9bc9-0b2a07234456, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:07.812578285Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fdc9e743-1329-4663-9bc9-0b2a07234456, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:07.812632936Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:07.833326301Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:07.833381233Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:07.853090231Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:07.853155482Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:07.853367786Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:07.853523998Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:07.949721909Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:07.9498049Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:07.949830911Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:07.949841191Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:07.949851245Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:07.949859204Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:07.949873698Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:07.949881949Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:07.949886542Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:07.949892655Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:07.949897792Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:07.949903042Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:07.949913642Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:07.949920702Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:07.949927332Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:07.949933802Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:07.949940122Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:07.949959323Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:07.949965643Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:07.950004934Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"d00b6cc5-af96-4008-a3b7-ed05279e9c27","remote_addr":"[fd00:1122:3344:101::1]:42546","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95\", TimeNotSynchronized), (\"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7\", TimeNotSynchronized), (\"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878\", TimeNotSynchronized), (\"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47\", TimeNotSynchronized), (\"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846\", TimeNotSynchronized), (\"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0\", TimeNotSynchronized), (\"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2\", TimeNotSynchronized), (\"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\", TimeNotSynchronized), (\"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae\", TimeNotSynchronized), (\"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868\", TimeNotSynchronized), (\"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c\", TimeNotSynchronized), (\"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a\", TimeNotSynchronized), (\"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759\", TimeNotSynchronized), (\"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff\", TimeNotSynchronized), (\"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab\", TimeNotSynchronized), (\"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42\", TimeNotSynchronized), (\"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456\", TimeNotSynchronized), (\"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af\", TimeNotSynchronized), (\"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911\", TimeNotSynchronized)]","latency_us":500956,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:07.950230088Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"1.172513085s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d00b6cc5-af96-4008-a3b7-ed05279e9c27\", \"content-length\": \"124\", \"date\": \"Mon, 26 Feb 2024 07:13:07 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d00b6cc5-af96-4008-a3b7-ed05279e9c27\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: d0d03c23-79c4-4c60-898e-739f188b5615, kind: Service { id: 3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc }, name: Name(\"ntp-3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 196, 59])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 4b099cd4-61db-4587-84b8-cf9015577106, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 4c8e127e-7304-49fa-9de2-79bbde180d94, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: d60f3c38-1add-47dd-baca-a6103e9614ae, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 864c4b26-c35e-42dc-a5b0-3e041205ccbc, kind: Service { id: d60f3c38-1add-47dd-baca-a6103e9614ae }, name: Name(\"external-dns-d60f3c38-1add-47dd-baca-a6103e9614ae\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 7])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: c9ac5e72-089e-4e66-8c75-d1c69a21aee1, kind: Service { id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95 }, name: Name(\"external-dns-740e2473-798e-4bf1-9bf5-7a2c9ce38b95\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 248])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 60944a67-2d69-42d2-8ce7-1215cdc06eab, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 850653ea-9272-4e6c-b3ae-b169328ac41c, kind: Service { id: 60944a67-2d69-42d2-8ce7-1215cdc06eab }, name: Name(\"nexus-60944a67-2d69-42d2-8ce7-1215cdc06eab\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 217, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 3c85fee5-2c8b-46c2-a723-6c621ecfb3be, kind: Service { id: 3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7 }, name: Name(\"nexus-3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 215])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e72f50ee-9b20-4af9-becc-46f37972a3b0, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 03f00562-72e1-4d42-8124-11bb07005618, kind: Service { id: e72f50ee-9b20-4af9-becc-46f37972a3b0 }, name: Name(\"nexus-e72f50ee-9b20-4af9-becc-46f37972a3b0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 176, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7fcbadc5-146c-4a16-bbe8-9fe02e4fe911, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: fbaa903b-4740-4365-b35e-68123cc1d3a2, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1693d50c-fcf2-4249-86ae-6e12a08ac868, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 065d24ba-ce47-47d1-9175-26eabb479878, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: bc1764f3-5054-4735-919d-fb5def7e7e42, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a493457b-acb1-4029-bcc4-1d8290fa8759, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 1daa94e0-684d-4a00-9332-60a34c24e94a, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: fdc9e743-1329-4663-9bc9-0b2a07234456, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:09.120417429Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:09.121083731Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:09.121098171Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:09.143890524Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:09.146227065Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:09.164781333Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:09.166708307Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:09.18500298Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:09.185056511Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:09.203736612Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:09.203793313Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b099cd4-61db-4587-84b8-cf9015577106, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:09.221916523Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4b099cd4-61db-4587-84b8-cf9015577106, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:09.221972294Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:09.239942271Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:09.240001842Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4c8e127e-7304-49fa-9de2-79bbde180d94, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:09.25912234Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4c8e127e-7304-49fa-9de2-79bbde180d94, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:09.259175721Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d60f3c38-1add-47dd-baca-a6103e9614ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:09.278744167Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d60f3c38-1add-47dd-baca-a6103e9614ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:09.278796458Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:09.298280272Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:09.298336413Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:09.318422679Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:09.318475819Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bc1764f3-5054-4735-919d-fb5def7e7e42, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:09.337543346Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bc1764f3-5054-4735-919d-fb5def7e7e42, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:09.337592907Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a493457b-acb1-4029-bcc4-1d8290fa8759, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:09.35641419Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a493457b-acb1-4029-bcc4-1d8290fa8759, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:09.356471001Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:09.374765544Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:09.374822735Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:09.393533726Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:09.393592047Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1daa94e0-684d-4a00-9332-60a34c24e94a, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:09.411587235Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1daa94e0-684d-4a00-9332-60a34c24e94a, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:09.411646596Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:09.4305615Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:09.430617581Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:09.449471144Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:09.449537136Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fdc9e743-1329-4663-9bc9-0b2a07234456, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:09.467751236Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fdc9e743-1329-4663-9bc9-0b2a07234456, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:09.467804689Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:09.486174893Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:09.486222254Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:09.504652147Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:09.504712971Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:09.504932475Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:09.505062047Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:09.598675372Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:09.598759473Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:09.598767603Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:09.598774833Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:09.598781714Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:09.598788883Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:09.598796054Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:09.598804564Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:09.598813494Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:09.598825724Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:09.598834514Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:09.598844544Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:09.598852865Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:09.598859765Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:09.598879445Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:09.598887115Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:09.598893785Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:09.598901965Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:09.598910296Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:09.598958346Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"8a15301b-d821-493e-b907-7967861ff9a3","remote_addr":"[fd00:1122:3344:101::1]:42546","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2\", TimeNotSynchronized), (\"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868\", TimeNotSynchronized), (\"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878\", TimeNotSynchronized), (\"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff\", TimeNotSynchronized), (\"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456\", TimeNotSynchronized), (\"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af\", TimeNotSynchronized), (\"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95\", TimeNotSynchronized), (\"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c\", TimeNotSynchronized), (\"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846\", TimeNotSynchronized), (\"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911\", TimeNotSynchronized), (\"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0\", TimeNotSynchronized), (\"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42\", TimeNotSynchronized), (\"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47\", TimeNotSynchronized), (\"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759\", TimeNotSynchronized), (\"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a\", TimeNotSynchronized), (\"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\", TimeNotSynchronized), (\"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae\", TimeNotSynchronized), (\"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab\", TimeNotSynchronized), (\"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7\", TimeNotSynchronized)]","latency_us":479361,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:09.599184181Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"1.973294757s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8a15301b-d821-493e-b907-7967861ff9a3\", \"content-length\": \"124\", \"date\": \"Mon, 26 Feb 2024 07:13:09 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"8a15301b-d821-493e-b907-7967861ff9a3\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: d0d03c23-79c4-4c60-898e-739f188b5615, kind: Service { id: 3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc }, name: Name(\"ntp-3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 196, 59])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 4b099cd4-61db-4587-84b8-cf9015577106, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 4c8e127e-7304-49fa-9de2-79bbde180d94, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: d60f3c38-1add-47dd-baca-a6103e9614ae, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 864c4b26-c35e-42dc-a5b0-3e041205ccbc, kind: Service { id: d60f3c38-1add-47dd-baca-a6103e9614ae }, name: Name(\"external-dns-d60f3c38-1add-47dd-baca-a6103e9614ae\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 7])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: c9ac5e72-089e-4e66-8c75-d1c69a21aee1, kind: Service { id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95 }, name: Name(\"external-dns-740e2473-798e-4bf1-9bf5-7a2c9ce38b95\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 248])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 60944a67-2d69-42d2-8ce7-1215cdc06eab, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 850653ea-9272-4e6c-b3ae-b169328ac41c, kind: Service { id: 60944a67-2d69-42d2-8ce7-1215cdc06eab }, name: Name(\"nexus-60944a67-2d69-42d2-8ce7-1215cdc06eab\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 217, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 3c85fee5-2c8b-46c2-a723-6c621ecfb3be, kind: Service { id: 3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7 }, name: Name(\"nexus-3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 215])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e72f50ee-9b20-4af9-becc-46f37972a3b0, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 03f00562-72e1-4d42-8124-11bb07005618, kind: Service { id: e72f50ee-9b20-4af9-becc-46f37972a3b0 }, name: Name(\"nexus-e72f50ee-9b20-4af9-becc-46f37972a3b0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 176, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7fcbadc5-146c-4a16-bbe8-9fe02e4fe911, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: fbaa903b-4740-4365-b35e-68123cc1d3a2, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1693d50c-fcf2-4249-86ae-6e12a08ac868, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 065d24ba-ce47-47d1-9175-26eabb479878, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: bc1764f3-5054-4735-919d-fb5def7e7e42, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a493457b-acb1-4029-bcc4-1d8290fa8759, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 1daa94e0-684d-4a00-9332-60a34c24e94a, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: fdc9e743-1329-4663-9bc9-0b2a07234456, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:11.568705479Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:11.569435692Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:11.569452532Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:11.591999361Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:11.592059842Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:11.613353478Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:11.613407309Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:11.632864663Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:11.632919809Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:11.652464819Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:11.65251722Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b099cd4-61db-4587-84b8-cf9015577106, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:11.670759862Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4b099cd4-61db-4587-84b8-cf9015577106, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:11.670818015Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:11.691195794Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:11.691256755Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4c8e127e-7304-49fa-9de2-79bbde180d94, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:11.709907565Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4c8e127e-7304-49fa-9de2-79bbde180d94, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:11.709958955Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d60f3c38-1add-47dd-baca-a6103e9614ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:11.729410369Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d60f3c38-1add-47dd-baca-a6103e9614ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:11.72947806Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:11.749365322Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:11.749418523Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:11.768309047Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:11.768366408Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bc1764f3-5054-4735-919d-fb5def7e7e42, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:11.787826951Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bc1764f3-5054-4735-919d-fb5def7e7e42, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:11.787879932Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a493457b-acb1-4029-bcc4-1d8290fa8759, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:11.806568263Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a493457b-acb1-4029-bcc4-1d8290fa8759, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:11.806617804Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:11.824765944Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:11.824821065Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:11.843338295Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:11.843392794Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1daa94e0-684d-4a00-9332-60a34c24e94a, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:11.861474143Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1daa94e0-684d-4a00-9332-60a34c24e94a, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:11.861531534Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:11.880351117Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:11.880411238Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:11.898553259Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:11.89860514Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fdc9e743-1329-4663-9bc9-0b2a07234456, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:11.91674308Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fdc9e743-1329-4663-9bc9-0b2a07234456, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:11.916795271Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:11.935484501Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:11.935528952Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:11.955110258Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:11.9551904Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:11.955383123Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:11.955529235Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:12.052719693Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:12.052790087Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:12.052799227Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:12.052805087Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:12.052810525Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:12.052816595Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:12.052823705Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:12.052829385Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:12.052836785Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:12.052843806Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:12.052849996Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:12.052881826Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:12.052893406Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:12.052899627Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:12.052904087Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:12.052908427Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:12.052914287Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:12.052919827Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:12.052925397Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:12.052986198Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"4de4c4d8-95b8-4e77-8fc9-c142cef23761","remote_addr":"[fd00:1122:3344:101::1]:42546","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab\", TimeNotSynchronized), (\"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878\", TimeNotSynchronized), (\"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47\", TimeNotSynchronized), (\"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911\", TimeNotSynchronized), (\"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846\", TimeNotSynchronized), (\"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456\", TimeNotSynchronized), (\"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\", TimeNotSynchronized), (\"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0\", TimeNotSynchronized), (\"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868\", TimeNotSynchronized), (\"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7\", TimeNotSynchronized), (\"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff\", TimeNotSynchronized), (\"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42\", TimeNotSynchronized), (\"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c\", TimeNotSynchronized), (\"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759\", TimeNotSynchronized), (\"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af\", TimeNotSynchronized), (\"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a\", TimeNotSynchronized), (\"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2\", TimeNotSynchronized), (\"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae\", TimeNotSynchronized), (\"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95\", TimeNotSynchronized)]","latency_us":485067,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:12.053212702Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"3.321975768s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4de4c4d8-95b8-4e77-8fc9-c142cef23761\", \"content-length\": \"124\", \"date\": \"Mon, 26 Feb 2024 07:13:11 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"4de4c4d8-95b8-4e77-8fc9-c142cef23761\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: d0d03c23-79c4-4c60-898e-739f188b5615, kind: Service { id: 3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc }, name: Name(\"ntp-3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 196, 59])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 4b099cd4-61db-4587-84b8-cf9015577106, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 4c8e127e-7304-49fa-9de2-79bbde180d94, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: d60f3c38-1add-47dd-baca-a6103e9614ae, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 864c4b26-c35e-42dc-a5b0-3e041205ccbc, kind: Service { id: d60f3c38-1add-47dd-baca-a6103e9614ae }, name: Name(\"external-dns-d60f3c38-1add-47dd-baca-a6103e9614ae\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 7])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: c9ac5e72-089e-4e66-8c75-d1c69a21aee1, kind: Service { id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95 }, name: Name(\"external-dns-740e2473-798e-4bf1-9bf5-7a2c9ce38b95\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 248])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 60944a67-2d69-42d2-8ce7-1215cdc06eab, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 850653ea-9272-4e6c-b3ae-b169328ac41c, kind: Service { id: 60944a67-2d69-42d2-8ce7-1215cdc06eab }, name: Name(\"nexus-60944a67-2d69-42d2-8ce7-1215cdc06eab\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 217, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 3c85fee5-2c8b-46c2-a723-6c621ecfb3be, kind: Service { id: 3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7 }, name: Name(\"nexus-3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 215])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e72f50ee-9b20-4af9-becc-46f37972a3b0, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 03f00562-72e1-4d42-8124-11bb07005618, kind: Service { id: e72f50ee-9b20-4af9-becc-46f37972a3b0 }, name: Name(\"nexus-e72f50ee-9b20-4af9-becc-46f37972a3b0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 176, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7fcbadc5-146c-4a16-bbe8-9fe02e4fe911, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: fbaa903b-4740-4365-b35e-68123cc1d3a2, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1693d50c-fcf2-4249-86ae-6e12a08ac868, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 065d24ba-ce47-47d1-9175-26eabb479878, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: bc1764f3-5054-4735-919d-fb5def7e7e42, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a493457b-acb1-4029-bcc4-1d8290fa8759, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 1daa94e0-684d-4a00-9332-60a34c24e94a, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: fdc9e743-1329-4663-9bc9-0b2a07234456, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:15.367817693Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:15.368463334Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:15.368473575Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:15.391888408Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:15.39196237Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:15.413735525Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:15.413814226Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:15.432098319Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:15.43215121Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:15.451521073Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:15.451584245Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b099cd4-61db-4587-84b8-cf9015577106, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:15.470114201Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4b099cd4-61db-4587-84b8-cf9015577106, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:15.470170532Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:15.490720865Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:15.490794387Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4c8e127e-7304-49fa-9de2-79bbde180d94, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:15.509269013Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4c8e127e-7304-49fa-9de2-79bbde180d94, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:15.509334424Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d60f3c38-1add-47dd-baca-a6103e9614ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:15.529300867Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d60f3c38-1add-47dd-baca-a6103e9614ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:15.529377338Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:15.54867269Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:15.548738081Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:15.568451709Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:15.56851075Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bc1764f3-5054-4735-919d-fb5def7e7e42, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:15.588835939Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bc1764f3-5054-4735-919d-fb5def7e7e42, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:15.5888928Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a493457b-acb1-4029-bcc4-1d8290fa8759, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:15.608541818Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a493457b-acb1-4029-bcc4-1d8290fa8759, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:15.608600309Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:15.627771128Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:15.627833209Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:15.64659546Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:15.646663241Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1daa94e0-684d-4a00-9332-60a34c24e94a, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:15.665877491Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1daa94e0-684d-4a00-9332-60a34c24e94a, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:15.665942212Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:15.684336067Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:15.684389868Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:15.702681271Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:15.702737562Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fdc9e743-1329-4663-9bc9-0b2a07234456, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:15.722746656Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fdc9e743-1329-4663-9bc9-0b2a07234456, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:15.722805297Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:15.741200172Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:15.741266163Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:15.759119419Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:15.7591866Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:15.759394154Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:15.759527026Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:15.855321669Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:15.855414391Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:15.855423301Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:15.855432701Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:15.855440161Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:15.855447111Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:15.855454792Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:15.855461952Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:15.855469802Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:15.855501771Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:15.855508961Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:15.855524684Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:15.855533309Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:15.855541833Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:15.855547363Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:15.855554093Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:15.855559523Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:15.855564843Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:15.855569683Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:15.855658115Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"260f546b-bc80-4f43-ba23-158dc5cc2f93","remote_addr":"[fd00:1122:3344:101::1]:42546","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868\", TimeNotSynchronized), (\"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff\", TimeNotSynchronized), (\"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95\", TimeNotSynchronized), (\"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0\", TimeNotSynchronized), (\"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\", TimeNotSynchronized), (\"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae\", TimeNotSynchronized), (\"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42\", TimeNotSynchronized), (\"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878\", TimeNotSynchronized), (\"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c\", TimeNotSynchronized), (\"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759\", TimeNotSynchronized), (\"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47\", TimeNotSynchronized), (\"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456\", TimeNotSynchronized), (\"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab\", TimeNotSynchronized), (\"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2\", TimeNotSynchronized), (\"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af\", TimeNotSynchronized), (\"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7\", TimeNotSynchronized), (\"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a\", TimeNotSynchronized), (\"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846\", TimeNotSynchronized), (\"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911\", TimeNotSynchronized)]","latency_us":488712,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:15.85591463Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"1.942935422s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"260f546b-bc80-4f43-ba23-158dc5cc2f93\", \"content-length\": \"124\", \"date\": \"Mon, 26 Feb 2024 07:13:15 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"260f546b-bc80-4f43-ba23-158dc5cc2f93\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:17.487583917Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:50399"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:17.487680489Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"99578173-74e8-4ec2-9f3c-998d5995fc1a","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:50399","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":42,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: d0d03c23-79c4-4c60-898e-739f188b5615, kind: Service { id: 3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc }, name: Name(\"ntp-3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 196, 59])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 4b099cd4-61db-4587-84b8-cf9015577106, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 4c8e127e-7304-49fa-9de2-79bbde180d94, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: d60f3c38-1add-47dd-baca-a6103e9614ae, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 864c4b26-c35e-42dc-a5b0-3e041205ccbc, kind: Service { id: d60f3c38-1add-47dd-baca-a6103e9614ae }, name: Name(\"external-dns-d60f3c38-1add-47dd-baca-a6103e9614ae\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 7])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: c9ac5e72-089e-4e66-8c75-d1c69a21aee1, kind: Service { id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95 }, name: Name(\"external-dns-740e2473-798e-4bf1-9bf5-7a2c9ce38b95\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 248])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 60944a67-2d69-42d2-8ce7-1215cdc06eab, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 850653ea-9272-4e6c-b3ae-b169328ac41c, kind: Service { id: 60944a67-2d69-42d2-8ce7-1215cdc06eab }, name: Name(\"nexus-60944a67-2d69-42d2-8ce7-1215cdc06eab\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 217, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 3c85fee5-2c8b-46c2-a723-6c621ecfb3be, kind: Service { id: 3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7 }, name: Name(\"nexus-3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 215])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e72f50ee-9b20-4af9-becc-46f37972a3b0, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 03f00562-72e1-4d42-8124-11bb07005618, kind: Service { id: e72f50ee-9b20-4af9-becc-46f37972a3b0 }, name: Name(\"nexus-e72f50ee-9b20-4af9-becc-46f37972a3b0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 176, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7fcbadc5-146c-4a16-bbe8-9fe02e4fe911, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: fbaa903b-4740-4365-b35e-68123cc1d3a2, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1693d50c-fcf2-4249-86ae-6e12a08ac868, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 065d24ba-ce47-47d1-9175-26eabb479878, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: bc1764f3-5054-4735-919d-fb5def7e7e42, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a493457b-acb1-4029-bcc4-1d8290fa8759, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 1daa94e0-684d-4a00-9332-60a34c24e94a, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: fdc9e743-1329-4663-9bc9-0b2a07234456, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:17.794685995Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:17.795727163Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:17.795738423Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:17.817501248Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:17.817561119Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:17.837450901Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:17.837507262Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:17.858865889Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:17.85892837Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:17.877058597Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:17.877122612Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b099cd4-61db-4587-84b8-cf9015577106, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:17.895663769Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4b099cd4-61db-4587-84b8-cf9015577106, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:17.89572686Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:17.914035964Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:17.914093975Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4c8e127e-7304-49fa-9de2-79bbde180d94, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:17.93192929Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4c8e127e-7304-49fa-9de2-79bbde180d94, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:17.932004702Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d60f3c38-1add-47dd-baca-a6103e9614ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:17.951278592Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d60f3c38-1add-47dd-baca-a6103e9614ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:17.951336443Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:17.969845221Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:17.969904251Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:17.988585592Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:17.988650203Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bc1764f3-5054-4735-919d-fb5def7e7e42, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:18.006982097Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bc1764f3-5054-4735-919d-fb5def7e7e42, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:18.007045868Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a493457b-acb1-4029-bcc4-1d8290fa8759, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:18.025917761Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a493457b-acb1-4029-bcc4-1d8290fa8759, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:18.025982293Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:18.044025032Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:18.044079482Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:18.062543223Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:18.06262381Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1daa94e0-684d-4a00-9332-60a34c24e94a, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:18.081119707Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1daa94e0-684d-4a00-9332-60a34c24e94a, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:18.081185998Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:18.099545663Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:18.099606494Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:18.119516886Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:18.119579667Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fdc9e743-1329-4663-9bc9-0b2a07234456, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:18.138090504Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fdc9e743-1329-4663-9bc9-0b2a07234456, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:18.138155945Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:18.15768248Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:18.157740931Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:18.177519431Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:18.177605782Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:18.177872767Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:18.17802083Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:18.273053089Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:18.273120661Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:18.273142561Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:18.273150651Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:18.273158091Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:18.273163911Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:18.273169891Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:18.273177491Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:18.273185002Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:18.273192372Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:18.273200122Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:18.273204562Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:18.273210092Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:18.273214162Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:18.273218242Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:18.273222872Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:18.273228783Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:18.273234363Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:18.273245503Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:18.273301214Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"df653990-170e-44bc-896d-998a44000a92","remote_addr":"[fd00:1122:3344:101::1]:42546","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\", TimeNotSynchronized), (\"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2\", TimeNotSynchronized), (\"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42\", TimeNotSynchronized), (\"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868\", TimeNotSynchronized), (\"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878\", TimeNotSynchronized), (\"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab\", TimeNotSynchronized), (\"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c\", TimeNotSynchronized), (\"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff\", TimeNotSynchronized), (\"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a\", TimeNotSynchronized), (\"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae\", TimeNotSynchronized), (\"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95\", TimeNotSynchronized), (\"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af\", TimeNotSynchronized), (\"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47\", TimeNotSynchronized), (\"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846\", TimeNotSynchronized), (\"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456\", TimeNotSynchronized), (\"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911\", TimeNotSynchronized), (\"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0\", TimeNotSynchronized), (\"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7\", TimeNotSynchronized), (\"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759\", TimeNotSynchronized)]","latency_us":479173,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:18.273509297Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"6.678234673s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"df653990-170e-44bc-896d-998a44000a92\", \"content-length\": \"124\", \"date\": \"Mon, 26 Feb 2024 07:13:17 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"df653990-170e-44bc-896d-998a44000a92\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: d0d03c23-79c4-4c60-898e-739f188b5615, kind: Service { id: 3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc }, name: Name(\"ntp-3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 196, 59])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 4b099cd4-61db-4587-84b8-cf9015577106, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 4c8e127e-7304-49fa-9de2-79bbde180d94, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: d60f3c38-1add-47dd-baca-a6103e9614ae, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 864c4b26-c35e-42dc-a5b0-3e041205ccbc, kind: Service { id: d60f3c38-1add-47dd-baca-a6103e9614ae }, name: Name(\"external-dns-d60f3c38-1add-47dd-baca-a6103e9614ae\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 7])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: c9ac5e72-089e-4e66-8c75-d1c69a21aee1, kind: Service { id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95 }, name: Name(\"external-dns-740e2473-798e-4bf1-9bf5-7a2c9ce38b95\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 248])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 60944a67-2d69-42d2-8ce7-1215cdc06eab, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 850653ea-9272-4e6c-b3ae-b169328ac41c, kind: Service { id: 60944a67-2d69-42d2-8ce7-1215cdc06eab }, name: Name(\"nexus-60944a67-2d69-42d2-8ce7-1215cdc06eab\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 217, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 3c85fee5-2c8b-46c2-a723-6c621ecfb3be, kind: Service { id: 3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7 }, name: Name(\"nexus-3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 215])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e72f50ee-9b20-4af9-becc-46f37972a3b0, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 03f00562-72e1-4d42-8124-11bb07005618, kind: Service { id: e72f50ee-9b20-4af9-becc-46f37972a3b0 }, name: Name(\"nexus-e72f50ee-9b20-4af9-becc-46f37972a3b0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 176, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7fcbadc5-146c-4a16-bbe8-9fe02e4fe911, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: fbaa903b-4740-4365-b35e-68123cc1d3a2, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1693d50c-fcf2-4249-86ae-6e12a08ac868, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 065d24ba-ce47-47d1-9175-26eabb479878, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: bc1764f3-5054-4735-919d-fb5def7e7e42, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a493457b-acb1-4029-bcc4-1d8290fa8759, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 1daa94e0-684d-4a00-9332-60a34c24e94a, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: fdc9e743-1329-4663-9bc9-0b2a07234456, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:24.935310735Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:24.936110109Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:24.93612019Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:24.959206148Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:24.959268129Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:24.979821422Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:24.979894863Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:25.000898735Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:25.000963006Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:25.02045335Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:25.020517321Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b099cd4-61db-4587-84b8-cf9015577106, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:25.039196282Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4b099cd4-61db-4587-84b8-cf9015577106, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:25.039252592Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:25.056738272Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:25.056815623Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4c8e127e-7304-49fa-9de2-79bbde180d94, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:25.07646859Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4c8e127e-7304-49fa-9de2-79bbde180d94, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:25.076534541Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d60f3c38-1add-47dd-baca-a6103e9614ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:25.094825985Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d60f3c38-1add-47dd-baca-a6103e9614ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:25.094887636Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:25.115237075Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:25.115302587Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:25.133124962Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:25.133191593Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bc1764f3-5054-4735-919d-fb5def7e7e42, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:25.153195016Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bc1764f3-5054-4735-919d-fb5def7e7e42, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:25.153261707Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a493457b-acb1-4029-bcc4-1d8290fa8759, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:25.171545611Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a493457b-acb1-4029-bcc4-1d8290fa8759, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:25.173900652Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:25.188964398Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:25.191339971Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:25.207332553Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:25.207390914Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1daa94e0-684d-4a00-9332-60a34c24e94a, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:25.226817858Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1daa94e0-684d-4a00-9332-60a34c24e94a, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:25.226876148Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:25.244643762Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:25.244700054Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:25.26311067Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:25.26317258Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fdc9e743-1329-4663-9bc9-0b2a07234456, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:25.280972795Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fdc9e743-1329-4663-9bc9-0b2a07234456, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:25.281037292Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:25.300021971Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:25.300077122Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:25.319027547Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:25.319101998Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:25.319285482Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:25.319427114Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:25.413546928Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:25.413623829Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:25.413632829Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:25.413638909Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:25.413646629Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:25.4136545Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:25.41366207Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:25.41366792Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:25.413675519Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:25.4136821Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:25.413687913Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:25.413702389Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:25.413708304Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:25.413717681Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:25.413723631Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:25.413736631Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:25.413743451Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:25.413750411Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:25.413755971Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:25.413810032Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"d6b66c74-a23d-4f5b-97f7-0a5a0206f35d","remote_addr":"[fd00:1122:3344:101::1]:42546","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846\", TimeNotSynchronized), (\"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff\", TimeNotSynchronized), (\"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c\", TimeNotSynchronized), (\"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\", TimeNotSynchronized), (\"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2\", TimeNotSynchronized), (\"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42\", TimeNotSynchronized), (\"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868\", TimeNotSynchronized), (\"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0\", TimeNotSynchronized), (\"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911\", TimeNotSynchronized), (\"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878\", TimeNotSynchronized), (\"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456\", TimeNotSynchronized), (\"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae\", TimeNotSynchronized), (\"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af\", TimeNotSynchronized), (\"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95\", TimeNotSynchronized), (\"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab\", TimeNotSynchronized), (\"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759\", TimeNotSynchronized), (\"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47\", TimeNotSynchronized), (\"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7\", TimeNotSynchronized), (\"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a\", TimeNotSynchronized)]","latency_us":479196,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:25.414020386Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"2.859683233s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d6b66c74-a23d-4f5b-97f7-0a5a0206f35d\", \"content-length\": \"124\", \"date\": \"Mon, 26 Feb 2024 07:13:25 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d6b66c74-a23d-4f5b-97f7-0a5a0206f35d\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: d0d03c23-79c4-4c60-898e-739f188b5615, kind: Service { id: 3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc }, name: Name(\"ntp-3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 196, 59])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 4b099cd4-61db-4587-84b8-cf9015577106, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 4c8e127e-7304-49fa-9de2-79bbde180d94, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: d60f3c38-1add-47dd-baca-a6103e9614ae, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 864c4b26-c35e-42dc-a5b0-3e041205ccbc, kind: Service { id: d60f3c38-1add-47dd-baca-a6103e9614ae }, name: Name(\"external-dns-d60f3c38-1add-47dd-baca-a6103e9614ae\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 7])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: c9ac5e72-089e-4e66-8c75-d1c69a21aee1, kind: Service { id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95 }, name: Name(\"external-dns-740e2473-798e-4bf1-9bf5-7a2c9ce38b95\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 248])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 60944a67-2d69-42d2-8ce7-1215cdc06eab, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 850653ea-9272-4e6c-b3ae-b169328ac41c, kind: Service { id: 60944a67-2d69-42d2-8ce7-1215cdc06eab }, name: Name(\"nexus-60944a67-2d69-42d2-8ce7-1215cdc06eab\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 217, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 3c85fee5-2c8b-46c2-a723-6c621ecfb3be, kind: Service { id: 3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7 }, name: Name(\"nexus-3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 215])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e72f50ee-9b20-4af9-becc-46f37972a3b0, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 03f00562-72e1-4d42-8124-11bb07005618, kind: Service { id: e72f50ee-9b20-4af9-becc-46f37972a3b0 }, name: Name(\"nexus-e72f50ee-9b20-4af9-becc-46f37972a3b0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 176, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7fcbadc5-146c-4a16-bbe8-9fe02e4fe911, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: fbaa903b-4740-4365-b35e-68123cc1d3a2, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1693d50c-fcf2-4249-86ae-6e12a08ac868, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 065d24ba-ce47-47d1-9175-26eabb479878, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: bc1764f3-5054-4735-919d-fb5def7e7e42, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a493457b-acb1-4029-bcc4-1d8290fa8759, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 1daa94e0-684d-4a00-9332-60a34c24e94a, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: fdc9e743-1329-4663-9bc9-0b2a07234456, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:28.267024719Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:28.268340872Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:28.268350862Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:28.291579883Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:28.291646434Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:28.312599375Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:28.312661616Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:28.331454488Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:28.331520429Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:28.351883619Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:28.35195188Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b099cd4-61db-4587-84b8-cf9015577106, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:28.370497838Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4b099cd4-61db-4587-84b8-cf9015577106, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:28.372815539Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:28.388447255Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:28.388514526Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4c8e127e-7304-49fa-9de2-79bbde180d94, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:28.407068524Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4c8e127e-7304-49fa-9de2-79bbde180d94, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:28.407131005Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d60f3c38-1add-47dd-baca-a6103e9614ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:28.426782262Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d60f3c38-1add-47dd-baca-a6103e9614ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:28.426850974Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:28.446384439Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:28.44645509Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:28.465791222Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:28.465857453Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bc1764f3-5054-4735-919d-fb5def7e7e42, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:28.484654945Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bc1764f3-5054-4735-919d-fb5def7e7e42, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:28.484715686Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a493457b-acb1-4029-bcc4-1d8290fa8759, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:28.503848735Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a493457b-acb1-4029-bcc4-1d8290fa8759, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:28.503911946Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:28.522508434Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:28.522568275Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:28.541418879Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:28.54147458Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1daa94e0-684d-4a00-9332-60a34c24e94a, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:28.559533549Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1daa94e0-684d-4a00-9332-60a34c24e94a, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:28.55959558Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:28.577531887Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:28.577603758Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:28.597896857Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:28.597953358Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fdc9e743-1329-4663-9bc9-0b2a07234456, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:28.616567367Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fdc9e743-1329-4663-9bc9-0b2a07234456, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:28.616634258Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:28.63543228Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:28.635487461Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:28.654772652Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:28.654860883Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:28.655036336Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:28.655172619Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:28.75087955Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:28.750964862Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:28.750977322Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:28.750986562Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:28.750993572Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:28.751017643Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:28.751026663Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:28.751033793Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:28.751042463Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:28.751049313Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:28.751054543Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:28.751060913Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:28.751073284Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:28.751080184Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:28.751084954Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:28.751091834Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:28.751098764Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:28.751106824Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:28.751118085Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:28.751171286Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"78381ff1-1a25-4ff9-b176-477a9c75d732","remote_addr":"[fd00:1122:3344:101::1]:42546","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\", TimeNotSynchronized), (\"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7\", TimeNotSynchronized), (\"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95\", TimeNotSynchronized), (\"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c\", TimeNotSynchronized), (\"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47\", TimeNotSynchronized), (\"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456\", TimeNotSynchronized), (\"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0\", TimeNotSynchronized), (\"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868\", TimeNotSynchronized), (\"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae\", TimeNotSynchronized), (\"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911\", TimeNotSynchronized), (\"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759\", TimeNotSynchronized), (\"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff\", TimeNotSynchronized), (\"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846\", TimeNotSynchronized), (\"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2\", TimeNotSynchronized), (\"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af\", TimeNotSynchronized), (\"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a\", TimeNotSynchronized), (\"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878\", TimeNotSynchronized), (\"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42\", TimeNotSynchronized), (\"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab\", TimeNotSynchronized)]","latency_us":485005,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:28.75144237Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"6.72552384s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"78381ff1-1a25-4ff9-b176-477a9c75d732\", \"content-length\": \"124\", \"date\": \"Mon, 26 Feb 2024 07:13:28 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"78381ff1-1a25-4ff9-b176-477a9c75d732\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: d0d03c23-79c4-4c60-898e-739f188b5615, kind: Service { id: 3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc }, name: Name(\"ntp-3c0cf7c7-56f8-4eaf-ac62-71dd1d5bb2dc\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 196, 59])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 4b099cd4-61db-4587-84b8-cf9015577106, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 4c8e127e-7304-49fa-9de2-79bbde180d94, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: d60f3c38-1add-47dd-baca-a6103e9614ae, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 864c4b26-c35e-42dc-a5b0-3e041205ccbc, kind: Service { id: d60f3c38-1add-47dd-baca-a6103e9614ae }, name: Name(\"external-dns-d60f3c38-1add-47dd-baca-a6103e9614ae\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 7])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: c9ac5e72-089e-4e66-8c75-d1c69a21aee1, kind: Service { id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95 }, name: Name(\"external-dns-740e2473-798e-4bf1-9bf5-7a2c9ce38b95\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 248])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 60944a67-2d69-42d2-8ce7-1215cdc06eab, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 850653ea-9272-4e6c-b3ae-b169328ac41c, kind: Service { id: 60944a67-2d69-42d2-8ce7-1215cdc06eab }, name: Name(\"nexus-60944a67-2d69-42d2-8ce7-1215cdc06eab\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 217, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 3c85fee5-2c8b-46c2-a723-6c621ecfb3be, kind: Service { id: 3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7 }, name: Name(\"nexus-3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 215])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e72f50ee-9b20-4af9-becc-46f37972a3b0, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 03f00562-72e1-4d42-8124-11bb07005618, kind: Service { id: e72f50ee-9b20-4af9-becc-46f37972a3b0 }, name: Name(\"nexus-e72f50ee-9b20-4af9-becc-46f37972a3b0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 176, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7fcbadc5-146c-4a16-bbe8-9fe02e4fe911, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: fbaa903b-4740-4365-b35e-68123cc1d3a2, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1693d50c-fcf2-4249-86ae-6e12a08ac868, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 065d24ba-ce47-47d1-9175-26eabb479878, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: bc1764f3-5054-4735-919d-fb5def7e7e42, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a493457b-acb1-4029-bcc4-1d8290fa8759, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 1daa94e0-684d-4a00-9332-60a34c24e94a, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: fdc9e743-1329-4663-9bc9-0b2a07234456, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:35.460588495Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:35.461367439Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2a4f4adb-368e-4ad8-9ca4-08f3408cecf7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:35.461383729Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:35.485305442Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 70de9569-4ef0-4038-88e6-16ed7c5649a0, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:35.485384244Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:35.507891291Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d7ff77b4-c6c4-4720-908c-7a0571f1e748, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:35.510340054Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:35.527523688Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 89b5827c-8fa9-4662-9e35-ddf672d463bc, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:35.527584889Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:35.545781701Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 11ee5b01-c0d0-44c9-b381-52b9b32cde90, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:35.545841872Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b099cd4-61db-4587-84b8-cf9015577106, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:35.566614169Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4b099cd4-61db-4587-84b8-cf9015577106, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:35.56667092Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:35.585040255Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: dab2ea04-af43-4f0f-8d35-5d21dc9d8253, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:35.585097876Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4c8e127e-7304-49fa-9de2-79bbde180d94, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:35.604072681Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4c8e127e-7304-49fa-9de2-79bbde180d94, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:35.604129222Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d60f3c38-1add-47dd-baca-a6103e9614ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:35.622991305Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d60f3c38-1add-47dd-baca-a6103e9614ae, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:35.623053906Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:35.643447347Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:35.643505438Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:35.662352051Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cee74d96-bec7-4f5b-a73e-6269d90f58a7, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:35.662413102Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bc1764f3-5054-4735-919d-fb5def7e7e42, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:35.683738769Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bc1764f3-5054-4735-919d-fb5def7e7e42, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:35.6837964Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a493457b-acb1-4029-bcc4-1d8290fa8759, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:35.703798133Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a493457b-acb1-4029-bcc4-1d8290fa8759, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:35.706103534Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:35.724374267Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3d64a4fc-53ba-4f08-9b11-25144a6b622c, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:35.724444048Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:35.744833729Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: e17963b5-b281-4a7e-8117-ab6cd04a0a47, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:35.74715137Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1daa94e0-684d-4a00-9332-60a34c24e94a, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:35.764576038Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1daa94e0-684d-4a00-9332-60a34c24e94a, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:35.764634659Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:35.783534913Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ab5b9d42-40cc-4445-b0f5-4f0ae4700846, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:35.783596414Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:35.803719309Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6a54d259-4fc5-4b60-a000-0fbfcf7975ff, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:35.80377841Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fdc9e743-1329-4663-9bc9-0b2a07234456, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:35.822383849Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: fdc9e743-1329-4663-9bc9-0b2a07234456, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:35.82244201Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:35.842019776Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6a44f502-32e2-41ae-8de2-2c1fca4407af, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:35.842090977Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:35.861545411Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:35.861622093Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:35.861806146Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:35.861951868Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:35.966264072Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:254"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:35.968321128Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:254"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:36.021541089Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:254"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:36.021887295Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:36.021934536Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:36.021945526Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:36.021976207Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:36.022038506Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:36.022054258Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:36.022067308Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:36.215550548Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:254"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:36.215864834Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:36.215884664Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:36.215892624Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:36.215911914Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:36.215948705Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:36.239395929Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:254"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:36.240062591Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:36.253131932Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Configuring new Omicron zone: oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:36.254228852Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:36.259456064Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:36.265335058Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:36.270683392Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:36.275968576Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:36.28133485Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:36.286495382Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:36.430291643Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:36.435999244Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:36.441742186Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:36.446889989Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:36.451458947Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:36.457436013Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:36.46235147Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Configuring new Omicron zone: oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:36.468225794Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:36.47381115Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:36.479239168Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:36.485351756Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:36.490525558Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:36.496442272Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:36.502445159Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:36.509008315Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:36.514753776Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:36.520290314Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:36.525341703Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:36.530134988Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:36.535947721Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:36.542083559Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:36.58568526Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:271","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:36.628651999Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:271","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:36.665141064Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:271","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:36.706253641Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:271","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:36.740817681Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:271","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:36.742234557Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:296"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:36.742246057Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:188","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:36.800399585Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte1\", ip: 172.30.1.5, mac: MacAddr6([168, 64, 37, 255, 140, 7]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 }, vnic: \"vopte1\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:36.800493916Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1271","zone_type":"external_dns"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:36.805004356Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:296"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:36.805021566Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:188","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:36.864131421Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte2\", ip: 172.30.2.5, mac: MacAddr6([168, 64, 37, 255, 217, 111]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte2\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:36.864207272Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1271","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:36.866860849Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:296"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:36.866881169Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:188","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:36.942644928Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte3\", ip: 172.30.2.7, mac: MacAddr6([168, 64, 37, 255, 176, 228]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte3\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:36.94271737Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1271","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:36.945469838Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:296"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:36.945497579Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:188","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:36.999586275Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte4\", ip: 172.30.2.6, mac: MacAddr6([168, 64, 37, 255, 145, 215]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte4\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:36.999659266Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1271","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:37.001623851Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:296"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:37.001659382Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:188","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:37.061418808Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte5\", ip: 172.30.1.6, mac: MacAddr6([168, 64, 37, 255, 160, 248]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 }, vnic: \"vopte5\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:37.061494829Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1271","zone_type":"external_dns"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.1.164,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:37.061970567Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:98"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.1.164,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:37.062002708Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:105"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.1.164,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:37.068258899Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:133"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:37.071064248Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.1.166,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:37.10284601Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:98"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.1.166,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:37.102926921Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:105"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.1.168,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:37.103240297Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:98"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.1.168,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:37.103252227Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:105"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.1.167,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:37.103602223Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:98"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.1.167,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:37.103611073Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:105"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.1.165,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:37.103820067Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:98"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.1.165,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:37.103829007Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:105"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.1.166,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:37.10399915Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:133"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.1.168,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:37.108439239Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:133"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.1.167,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:37.111203918Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:133"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:37.111224878Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:37.111236098Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.1.165,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:37.113655651Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:133"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:37.116164085Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:37.116185526Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Configuring new Omicron zone: oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:37.244680057Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:37.251828133Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:37.258617873Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:37.266445671Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:37.273431535Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:37.279239397Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:37.284905917Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:37.291880051Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:37.298311825Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:37.304548837Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:40.442476243Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:40.442682247Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:40.46265062Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:40.462858834Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:40.533341389Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:40.533508742Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:40.826535241Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:40.826757445Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:40.834525252Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:40.834782956Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:40.83503837Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:40.835145293Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:40.841406064Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:40.847397639Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:40.861846285Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:40.862083299Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:40.86214359Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:40.867532845Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:40.90855348Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:40.90882396Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:40.908860216Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:40.916259996Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:40.922549378Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:40.922759541Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868\") failed: Property not found. retry in 69.313109ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:41.301621937Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878\") failed: Property not found. retry in 31.375703ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:41.323115117Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846\") failed: Property not found. retry in 37.585244ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:41.414667555Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2\") failed: Property not found. retry in 35.337176ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:46.39866879Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff\") failed: Property not found. retry in 38.843515ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:46.811506487Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:46.813117825Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759\") failed: Property not found. retry in 45.320969ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:46.82754669Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878\") failed: Property not found. retry in 118.071451ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:46.841197812Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846\") failed: Property not found. retry in 139.112183ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:46.856155816Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a\") failed: Property not found. retry in 56.804357ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:46.876963154Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868\") failed: Property not found. retry in 123.349083ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:46.893033628Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42\") failed: Property not found. retry in 49.911886ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:46.904233136Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2\") failed: Property not found. retry in 121.111872ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:46.913477389Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff\") failed: Property not found. retry in 95.30422ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:46.922200053Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456\") failed: Property not found. retry in 55.333495ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:46.933713857Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af\") failed: Property not found. retry in 49.942282ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:46.947843616Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c\") failed: Property not found. retry in 48.285715ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:46.960763035Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:46.961162592Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759\") failed: Property not found. retry in 57.261152ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:46.978078061Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878\") failed: Property not found. retry in 261.804981ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:46.988867092Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a\") failed: Property not found. retry in 119.622774ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:47.004792353Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42\") failed: Property not found. retry in 118.526431ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:47.01650938Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456\") failed: Property not found. retry in 87.011459ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:47.028433181Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af\") failed: Property not found. retry in 66.197698ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:47.039472876Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c\") failed: Property not found. retry in 96.099831ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:47.052491036Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759\") failed: Property not found. retry in 256.811623ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:47.065000717Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47\") failed: Property not found. retry in 66.494699ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:47.522252108Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878\") failed: Property not found. retry in 588.548549ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:47.535921612Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846\") failed: Property not found. retry in 297.622101ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:47.549279066Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a\") failed: Property not found. retry in 236.583579ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:47.563427926Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868\") failed: Property not found. retry in 214.591475ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:47.577586976Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42\") failed: Property not found. retry in 179.326482ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:47.590681858Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2\") failed: Property not found. retry in 157.718339ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:47.605810295Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff\") failed: Property not found. retry in 140.876489ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:47.620107868Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456\") failed: Property not found. retry in 112.754576ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:47.632749391Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af\") failed: Property not found. retry in 183.441331ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:47.64685111Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c\") failed: Property not found. retry in 208.8059ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:47.660198296Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759\") failed: Property not found. retry in 229.148872ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:47.674616851Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47\") failed: Property not found. retry in 61.826684ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:47.687769733Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456\") failed: Property not found. retry in 428.580258ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:47.76338737Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47\") failed: Property not found. retry in 220.707417ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:47.776302708Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42\") failed: Property not found. retry in 571.523684ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:47.790186724Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2\") failed: Property not found. retry in 564.313064ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:47.804513167Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff\") failed: Property not found. retry in 501.006575ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:47.817629869Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a\") failed: Property not found. retry in 532.222962ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:47.831951892Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868\") failed: Property not found. retry in 237.949458ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:47.845357889Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af\") failed: Property not found. retry in 235.446644ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:47.859211204Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846\") failed: Property not found. retry in 262.220849ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:47.874468423Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c\") failed: Property not found. retry in 288.077917ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:47.88726399Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759\") failed: Property not found. retry in 1.071415967s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:47.9172542Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47\") failed: Property not found. retry in 235.79758ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:48.011057497Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868\") failed: Property not found. retry in 737.870832ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:48.099389988Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af\") failed: Property not found. retry in 699.052117ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:48.112368948Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878\") failed: Property not found. retry in 862.183016ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:48.139179462Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846\") failed: Property not found. retry in 849.301834ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:48.152753712Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c\") failed: Property not found. retry in 1.138674203s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:48.189025953Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456\") failed: Property not found. retry in 950.225469ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:48.204470736Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47\") failed: Property not found. retry in 1.176717685s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:48.259098121Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff\") failed: Property not found. retry in 453.524753ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:48.331323918Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42\") failed: Property not found. retry in 1.172568389s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:48.37331941Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a\") failed: Property not found. retry in 615.482587ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:48.386398911Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2\") failed: Property not found. retry in 603.755328ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:48.397125151Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff\") failed: Property not found. retry in 1.366905572s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:48.796873406Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af\") failed: Property not found. retry in 1.149203604s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:48.82260142Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868\") failed: Property not found. retry in 889.08298ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:48.845491005Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759\") failed: Property not found. retry in 513.259995ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:48.996431933Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878\") failed: Property not found. retry in 871.500128ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:49.012470486Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846\") failed: Property not found. retry in 1.381640562s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:49.019732154Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2\") failed: Property not found. retry in 705.670681ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:49.027904541Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a\") failed: Property not found. retry in 1.313471643s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:49.038569237Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456\") failed: Property not found. retry in 837.365811ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:49.387295101Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c\") failed: Property not found. retry in 1.109249445s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:49.399304853Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911\") failed: Property not found. retry in 70.494288ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:49.410008532Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47\") failed: Property not found. retry in 1.344444888s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:49.443521414Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911\") failed: Property not found. retry in 103.562402ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:49.492266616Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759\") failed: Property not found. retry in 1.390572446s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:49.522790305Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42\") failed: Property not found. retry in 1.440388106s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:49.555566874Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911\") failed: Property not found. retry in 271.624807ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:49.607115895Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2\") failed: Property not found. retry in 948.262514ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:50.023221674Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911\") failed: Property not found. retry in 357.065237ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:50.038224914Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878\") failed: Property not found. retry in 1.411996766s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:50.050333969Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868\") failed: Property not found. retry in 1.233461843s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:50.061493386Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af\") failed: Property not found. retry in 1.308248429s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:50.219067601Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff\") failed: Property not found. retry in 507.42861ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:50.231339998Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456\") failed: Property not found. retry in 1.255131176s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:50.311498824Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a\") failed: Property not found. retry in 1.416530915s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:50.360374198Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911\") failed: Property not found. retry in 488.840444ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:50.406632426Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846\") failed: Property not found. retry in 1.372037368s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:50.48494255Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c\") failed: Property not found. retry in 507.469112ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:50.517352893Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff\") failed: Property not found. retry in 1.247679035s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:50.749382033Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47\") failed: Property not found. retry in 661.268632ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:50.920513998Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911\") failed: Property not found. retry in 654.851059ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:50.930892511Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759\") failed: Property not found. retry in 732.983996ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:50.940818977Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2\") failed: Property not found. retry in 1.440807202s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:50.981198731Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42\") failed: Property not found. retry in 807.506143ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:51.016398702Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c\") failed: Property not found. retry in 1.121268507s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:51.035769523Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868\") failed: Property not found. retry in 1.045359058s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:51.305263313Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878\") failed: Property not found. retry in 1.255649972s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:51.663108551Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911\") failed: Property not found. retry in 747.606106ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:51.676015028Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759\") failed: Property not found. retry in 838.091819ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:51.687719064Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47\") failed: Property not found. retry in 855.514862ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:51.699243427Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456\") failed: Property not found. retry in 1.477944647s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:51.710480505Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af\") failed: Property not found. retry in 660.653116ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:51.751216743Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a\") failed: Property not found. retry in 872.372441ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:51.787685036Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42\") failed: Property not found. retry in 655.325744ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:51.903409796Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846\") failed: Property not found. retry in 575.473074ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:51.915987068Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff\") failed: Property not found. retry in 797.743829ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:52.008633591Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c\") failed: Property not found. retry in 997.156296ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:52.169312283Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868\") failed: Property not found. retry in 1.005979716s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:52.363962034Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af\") failed: Property not found. retry in 1.139450876s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:52.42505371Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911\") failed: Property not found. retry in 1.248023599s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:52.437552151Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2\") failed: Property not found. retry in 1.278795535s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:52.448329811Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846\") failed: Property not found. retry in 1.17663076s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:52.499707606Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759\") failed: Property not found. retry in 674.527871ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:52.534706463Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47\") failed: Property not found. retry in 1.372422967s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:52.562942701Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42\") failed: Property not found. retry in 1.057072706s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:52.572090672Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a\") failed: Property not found. retry in 1.331855898s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:52.67067927Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff\") failed: Property not found. retry in 1.417958068s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:52.815105795Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878\") failed: Property not found. retry in 692.758705ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:52.988510262Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c\") failed: Property not found. retry in 768.203274ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:53.176025867Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456\") failed: Property not found. retry in 610.849407ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:53.19774829Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759\") failed: Property not found. retry in 1.357759162s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:53.218036837Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868\") failed: Property not found. retry in 1.007969493s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:53.380395379Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af\") failed: Property not found. retry in 1.139341482s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:53.573094876Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42\") failed: Property not found. retry in 888.818273ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:53.640235239Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846\") failed: Property not found. retry in 665.381418ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:53.686511865Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911\") failed: Property not found. retry in 1.356148373s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:53.695572794Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878\") failed: Property not found. retry in 1.444708278s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:53.704702725Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2\") failed: Property not found. retry in 1.147854163s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:53.736136829Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456\") failed: Property not found. retry in 678.399505ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:53.818271407Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47\") failed: Property not found. retry in 934.776167ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:53.944539753Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c\") failed: Property not found. retry in 767.755739ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:53.952616115Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a\") failed: Property not found. retry in 591.754136ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:54.011601755Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff\") failed: Property not found. retry in 717.622896ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:54.243032224Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846\") failed: Property not found. retry in 1.049647449s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:54.361217427Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868\") failed: Property not found. retry in 1.354624115s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:54.397646039Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456\") failed: Property not found. retry in 1.120454466s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:54.514515179Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42\") failed: Property not found. retry in 900.67414ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:54.542739876Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759\") failed: Property not found. retry in 628.554534ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:54.601654434Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a\") failed: Property not found. retry in 509.228043ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:54.634634796Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af\") failed: Property not found. retry in 1.073044542s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:54.723438501Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c\") failed: Property not found. retry in 695.685966ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:54.738407655Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:54.819718038Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:54.819933442Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47\") failed: Property not found. retry in 1.077352082s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:54.888604602Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2\") failed: Property not found. retry in 1.206979979s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:54.89756704Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff\") failed: Property not found. retry in 1.112978762s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:54.969407267Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95\") failed: Property not found. retry in 72.346066ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:55.003759782Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911\") failed: Property not found. retry in 1.325525122s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:55.063008646Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95\") failed: Property not found. retry in 140.646912ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:55.086430539Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a\") failed: Property not found. retry in 552.628054ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:55.154888886Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878\") failed: Property not found. retry in 689.946732ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:55.16419236Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95\") failed: Property not found. retry in 244.028034ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:55.238651342Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759\") failed: Property not found. retry in 1.114625863s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:55.247342945Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:55.343331857Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:55.34916022Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846\") failed: Property not found. retry in 1.403933694s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:55.420460727Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c\") failed: Property not found. retry in 1.140380856s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:55.444990349Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42\") failed: Property not found. retry in 881.420363ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:55.453419238Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95\") failed: Property not found. retry in 379.171284ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:55.492010718Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0\") failed: Property not found. retry in 68.771963ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:55.520222655Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab\") failed: Property not found. retry in 36.183465ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:55.53242123Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab\") failed: Property not found. retry in 123.719473ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:55.577395003Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0\") failed: Property not found. retry in 57.156913ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:55.597978236Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456\") failed: Property not found. retry in 1.398597429s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:55.644408994Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0\") failed: Property not found. retry in 298.787392ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:55.663760045Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab\") failed: Property not found. retry in 114.375942ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:55.711285193Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a\") failed: Property not found. retry in 1.30069967s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:55.718829206Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868\") failed: Property not found. retry in 550.254465ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:55.761374166Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af\") failed: Property not found. retry in 1.432752737s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:55.805533364Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab\") failed: Property not found. retry in 317.145385ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:55.833934485Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878\") failed: Property not found. retry in 1.48743196s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:55.863511896Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95\") failed: Property not found. retry in 410.663326ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:55.880454814Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0\") failed: Property not found. retry in 582.422104ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:55.975289066Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47\") failed: Property not found. retry in 966.609286ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:55.983261577Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff\") failed: Property not found. retry in 812.676396ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:56.105717835Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2\") failed: Property not found. retry in 599.96169ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:56.132011419Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab\") failed: Property not found. retry in 910.4263ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:56.164623573Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95\") failed: Property not found. retry in 866.095971ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:56.301296052Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868\") failed: Property not found. retry in 629.138973ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:56.320851257Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42\") failed: Property not found. retry in 1.021586594s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:56.344230129Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759\") failed: Property not found. retry in 1.017438738s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:56.372283683Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911\") failed: Property not found. retry in 793.409059ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:56.399378021Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0\") failed: Property not found. retry in 1.053445142s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:56.569931767Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c\") failed: Property not found. retry in 610.570538ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:56.596592307Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2\") failed: Property not found. retry in 1.09691503s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:56.742635451Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:56.771556381Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:13:56.771718203Z","hostname":"buskin","pid":1671,"zone":"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846\") failed: Property not found. retry in 807.515135ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:56.834022672Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff\") failed: Property not found. retry in 1.241694957s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:56.9270023Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47\") failed: Property not found. retry in 902.106654ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:56.960259877Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868\") failed: Property not found. retry in 538.367111ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:56.969708843Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7\") failed: Property not found. retry in 34.87751ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:56.983771911Z","hostname":"buskin","pid":1671,"zone":"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7\") failed: Property not found. retry in 128.256388ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:57.025849273Z","hostname":"buskin","pid":1671,"zone":"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a\") failed: Property not found. retry in 523.169837ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:57.033839884Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456\") failed: Property not found. retry in 903.107841ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:57.051750969Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab\") failed: Property not found. retry in 1.1565429s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:57.083186064Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7\") failed: Property not found. retry in 179.590208ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:57.163854035Z","hostname":"buskin","pid":1671,"zone":"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95\") failed: Property not found. retry in 1.241113819s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:57.175502501Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911\") failed: Property not found. retry in 805.345769ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:57.200192896Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c\") failed: Property not found. retry in 1.433022347s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:57.215397364Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af\") failed: Property not found. retry in 986.220654ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:57.247555511Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7\") failed: Property not found. retry in 412.334785ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:57.35249055Z","hostname":"buskin","pid":1671,"zone":"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878\") failed: Property not found. retry in 581.004772ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:57.361913016Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42\") failed: Property not found. retry in 1.355103495s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:57.375019697Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759\") failed: Property not found. retry in 1.104619521s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:57.399615911Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868\") failed: Property not found. retry in 1.10915779s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:57.517145793Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a\") failed: Property not found. retry in 594.949872ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:57.572379526Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0\") failed: Property not found. retry in 1.0312033s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:57.637651116Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846\") failed: Property not found. retry in 1.222895973s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:57.653240811Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7\") failed: Property not found. retry in 1.112467159s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:57.782690683Z","hostname":"buskin","pid":1671,"zone":"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2\") failed: Property not found. retry in 1.188170025s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:57.866721514Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47\") failed: Property not found. retry in 1.069372521s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:57.891207155Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878\") failed: Property not found. retry in 859.367093ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:57.957769009Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456\") failed: Property not found. retry in 887.265192ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:57.971543271Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911\") failed: Property not found. retry in 1.195976729s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:58.020575936Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a\") failed: Property not found. retry in 735.465668ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:58.17797728Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff\") failed: Property not found. retry in 965.463463ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:58.187458607Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af\") failed: Property not found. retry in 1.24326913s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:58.245283556Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab\") failed: Property not found. retry in 620.409076ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:58.253395369Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95\") failed: Property not found. retry in 1.333040564s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:58.426170275Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759\") failed: Property not found. retry in 1.32560938s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:58.513628356Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868\") failed: Property not found. retry in 1.491710364s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:58.651899083Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c\") failed: Property not found. retry in 819.022722ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:58.66646961Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0\") failed: Property not found. retry in 917.430603ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:58.68292693Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42\") failed: Property not found. retry in 974.028413ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:58.744964444Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878\") failed: Property not found. retry in 1.448665117s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:58.830017843Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456\") failed: Property not found. retry in 1.081487774s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:58.870340604Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab\") failed: Property not found. retry in 1.287884628s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:58.88545308Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846\") failed: Property not found. retry in 1.021023912s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:58.894383047Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7\") failed: Property not found. retry in 1.234400722s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:58.904402804Z","hostname":"buskin","pid":1671,"zone":"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a\") failed: Property not found. retry in 1.010323118s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:58.922617025Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47\") failed: Property not found. retry in 561.592361ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:58.9699988Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2\") failed: Property not found. retry in 1.090734193s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:59.06589429Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff\") failed: Property not found. retry in 739.694573ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:59.163776026Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911\") failed: Property not found. retry in 622.173405ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:59.225932591Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c\") failed: Property not found. retry in 1.494276749s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:59.494892052Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af\") failed: Property not found. retry in 527.242042ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:59.503399072Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47\") failed: Property not found. retry in 823.843651ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:59.540676409Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0\") failed: Property not found. retry in 820.789376ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:59.609361839Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42\") failed: Property not found. retry in 1.452198405s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:59.729324144Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95\") failed: Property not found. retry in 757.287867ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:59.768175828Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759\") failed: Property not found. retry in 937.588352ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:59.848037046Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911\") failed: Property not found. retry in 1.223943378s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:59.856990874Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff\") failed: Property not found. retry in 1.190124226s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:59.912621154Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846\") failed: Property not found. retry in 773.950594ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:59.923609778Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a\") failed: Property not found. retry in 733.61642ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:59.941943911Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456\") failed: Property not found. retry in 1.288407046s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:13:59.961402904Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af\") failed: Property not found. retry in 1.468691278s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:00.040511798Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7\") failed: Property not found. retry in 638.368471ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:00.149568001Z","hostname":"buskin","pid":1671,"zone":"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868\") failed: Property not found. retry in 1.211883392s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:00.158258424Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2\") failed: Property not found. retry in 1.135187136s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:00.166861305Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab\") failed: Property not found. retry in 673.190725ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:00.181815449Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878\") failed: Property not found. retry in 881.139628ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:00.291270809Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47\") failed: Property not found. retry in 608.810789ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:00.38837578Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0\") failed: Property not found. retry in 875.186308ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:00.444160973Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:00.468642185Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:00.46950096Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95\") failed: Property not found. retry in 518.383697ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:00.53928492Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a\") failed: Property not found. retry in 1.200918945s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:00.689346795Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846\") failed: Property not found. retry in 1.370690072s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:00.710318854Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae\") failed: Property not found. retry in 44.989411ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:00.726124313Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae\") failed: Property not found. retry in 137.982716ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:00.781710143Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759\") failed: Property not found. retry in 1.112174626s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:00.795196Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7\") failed: Property not found. retry in 1.28749865s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:00.804540515Z","hostname":"buskin","pid":1671,"zone":"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab\") failed: Property not found. retry in 1.121893292s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:00.865589091Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae\") failed: Property not found. retry in 218.602044ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:00.936393799Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c\") failed: Property not found. retry in 534.914674ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:01.001337514Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47\") failed: Property not found. retry in 971.522818ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:01.011784098Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95\") failed: Property not found. retry in 1.235806637s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:01.068977216Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911\") failed: Property not found. retry in 912.376877ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:01.092503171Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff\") failed: Property not found. retry in 509.843562ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:01.112922971Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae\") failed: Property not found. retry in 498.09461ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:01.164576671Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878\") failed: Property not found. retry in 1.099004999s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:01.182404775Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42\") failed: Property not found. retry in 626.862437ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:01.192090976Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456\") failed: Property not found. retry in 1.2881336s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:01.261837985Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2\") failed: Property not found. retry in 1.439260741s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:01.311502001Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0\") failed: Property not found. retry in 510.960753ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:01.328777795Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868\") failed: Property not found. retry in 516.093837ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:01.381155679Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af\") failed: Property not found. retry in 822.595883ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:01.551766115Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c\") failed: Property not found. retry in 1.005548518s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:01.564268066Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff\") failed: Property not found. retry in 1.263193438s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:01.706749067Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae\") failed: Property not found. retry in 974.552851ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:01.734120569Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42\") failed: Property not found. retry in 794.312249ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:01.829985809Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0\") failed: Property not found. retry in 1.213669641s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:01.849355561Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a\") failed: Property not found. retry in 1.283192673s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:01.900705426Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868\") failed: Property not found. retry in 693.893537ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:01.909039823Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759\") failed: Property not found. retry in 1.072033082s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:01.916567215Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47\") failed: Property not found. retry in 631.977372ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:01.991444135Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab\") failed: Property not found. retry in 1.045434602s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:01.999780662Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911\") failed: Property not found. retry in 1.432903783s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:02.013716718Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846\") failed: Property not found. retry in 1.02818977s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:02.089479563Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7\") failed: Property not found. retry in 822.139586ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:02.102506383Z","hostname":"buskin","pid":1671,"zone":"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878\") failed: Property not found. retry in 1.292254686s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:02.29214985Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95\") failed: Property not found. retry in 1.005436166s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:02.315720611Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af\") failed: Property not found. retry in 1.375337216s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:02.384133837Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:02.506607216Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456\") failed: Property not found. retry in 708.254186ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:02.564961854Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c\") failed: Property not found. retry in 1.008035042s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:02.5812123Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868\") failed: Property not found. retry in 885.471732ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:02.613416348Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47\") failed: Property not found. retry in 1.42057897s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:02.633417871Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42\") failed: Property not found. retry in 1.497634144s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:02.641118466Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\") failed: Property not found. retry in 38.745871ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:02.698338055Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae\") failed: Property not found. retry in 1.103018478s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:02.716927763Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\") failed: Property not found. retry in 108.166883ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:02.744696922Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2\") failed: Property not found. retry in 524.524936ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:02.759269969Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\") failed: Property not found. retry in 171.159391ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:02.865756916Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7\") failed: Property not found. retry in 1.040538934s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:02.933266506Z","hostname":"buskin","pid":1671,"zone":"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff\") failed: Property not found. retry in 1.046042677s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:02.976564209Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759\") failed: Property not found. retry in 926.180653ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:02.996056032Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\") failed: Property not found. retry in 237.466351ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:03.047651292Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab\") failed: Property not found. retry in 1.09628297s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:03.056482408Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0\") failed: Property not found. retry in 1.25814502s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:03.072814445Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846\") failed: Property not found. retry in 501.933761ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:03.126046264Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a\") failed: Property not found. retry in 904.509397ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:03.193146036Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456\") failed: Property not found. retry in 514.412175ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:03.282572953Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2\") failed: Property not found. retry in 1.418281416s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:03.293577296Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\") failed: Property not found. retry in 545.703951ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:03.301204101Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95\") failed: Property not found. retry in 866.032934ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:03.330255913Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911\") failed: Property not found. retry in 890.771155ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:03.455310527Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868\") failed: Property not found. retry in 940.033576ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:03.509959221Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878\") failed: Property not found. retry in 1.098151178s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:03.594956828Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c\") failed: Property not found. retry in 1.061805603s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:03.603396767Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846\") failed: Property not found. retry in 657.955547ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:03.638128689Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af\") failed: Property not found. retry in 526.833763ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:03.76864506Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456\") failed: Property not found. retry in 619.998478ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:03.806551558Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae\") failed: Property not found. retry in 1.004692512s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:03.82882134Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\") failed: Property not found. retry in 944.232591ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:03.855023382Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759\") failed: Property not found. retry in 1.432174878s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:03.929907872Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7\") failed: Property not found. retry in 774.497285ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:03.983191421Z","hostname":"buskin","pid":1671,"zone":"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff\") failed: Property not found. retry in 1.271056314s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:04.033160842Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47\") failed: Property not found. retry in 1.356631007s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:04.064694708Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a\") failed: Property not found. retry in 654.31034ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:04.106596076Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42\") failed: Property not found. retry in 1.412648754s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:04.149538703Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab\") failed: Property not found. retry in 791.364339ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:04.162775676Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95\") failed: Property not found. retry in 1.249733835s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:04.205398458Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af\") failed: Property not found. retry in 989.414074ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:04.305871359Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846\") failed: Property not found. retry in 873.406442ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:04.315393867Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0\") failed: Property not found. retry in 1.136602526s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:04.342416963Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911\") failed: Property not found. retry in 1.064539536s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:04.355819019Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456\") failed: Property not found. retry in 715.974184ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:04.436539042Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868\") failed: Property not found. retry in 1.117774441s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:04.459242192Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c\") failed: Property not found. retry in 1.429824603s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:04.677781424Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878\") failed: Property not found. retry in 1.01359727s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:04.702818695Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2\") failed: Property not found. retry in 982.660305ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:04.721499684Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7\") failed: Property not found. retry in 1.011845647s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:04.768122356Z","hostname":"buskin","pid":1671,"zone":"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a\") failed: Property not found. retry in 661.414844ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:04.776940552Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\") failed: Property not found. retry in 946.509715ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:04.807952408Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae\") failed: Property not found. retry in 1.495570786s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:04.842244152Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab\") failed: Property not found. retry in 874.097238ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:04.968547979Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456\") failed: Property not found. retry in 813.270405ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:05.162829323Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846\") failed: Property not found. retry in 1.2203773s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:05.198142096Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af\") failed: Property not found. retry in 620.563381ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:05.305375086Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff\") failed: Property not found. retry in 1.437639669s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:05.313856245Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759\") failed: Property not found. retry in 1.004612021s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:05.373732681Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911\") failed: Property not found. retry in 1.180999874s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:05.430012523Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47\") failed: Property not found. retry in 1.092119081s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:05.439014981Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a\") failed: Property not found. retry in 769.154898ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:05.447308988Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95\") failed: Property not found. retry in 961.949542ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:05.465461037Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0\") failed: Property not found. retry in 990.418959ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:05.491164541Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42\") failed: Property not found. retry in 722.996083ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:05.573287968Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868\") failed: Property not found. retry in 1.005030301s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:05.588686209Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2\") failed: Property not found. retry in 1.276443455s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:05.714568848Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878\") failed: Property not found. retry in 1.327111412s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:05.727946574Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\") failed: Property not found. retry in 1.33499334s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:05.766726567Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7\") failed: Property not found. retry in 985.166688ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:05.793222094Z","hostname":"buskin","pid":1671,"zone":"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab\") failed: Property not found. retry in 1.277054366s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:05.855791337Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af\") failed: Property not found. retry in 532.835353ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:05.945287605Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456\") failed: Property not found. retry in 1.080775459s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:05.989602486Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c\") failed: Property not found. retry in 517.865809ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:06.12033531Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a\") failed: Property not found. retry in 1.413829619s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:06.228826132Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42\") failed: Property not found. retry in 667.424697ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:06.308732511Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae\") failed: Property not found. retry in 903.015468ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:06.350767872Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759\") failed: Property not found. retry in 1.384342511s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:06.388374635Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846\") failed: Property not found. retry in 748.011226ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:06.429486089Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95\") failed: Property not found. retry in 1.327685606s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:06.439509596Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af\") failed: Property not found. retry in 807.682189ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:06.489484117Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0\") failed: Property not found. retry in 1.25947757s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:06.498306882Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47\") failed: Property not found. retry in 915.21564ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:06.542254877Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868\") failed: Property not found. retry in 613.826466ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:06.60543558Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911\") failed: Property not found. retry in 1.367166278s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:06.622090114Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c\") failed: Property not found. retry in 832.386904ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:06.649047449Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff\") failed: Property not found. retry in 1.188180004s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:06.762493178Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7\") failed: Property not found. retry in 1.374550383s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:06.791176104Z","hostname":"buskin","pid":1671,"zone":"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42\") failed: Property not found. retry in 1.119404602s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:06.988115305Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2\") failed: Property not found. retry in 787.92263ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:07.003860223Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878\") failed: Property not found. retry in 590.123897ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:07.067089487Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456\") failed: Property not found. retry in 966.706858ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:07.080147857Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\") failed: Property not found. retry in 1.00252589s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:07.114741237Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab\") failed: Property not found. retry in 1.128001943s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:07.150907265Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846\") failed: Property not found. retry in 1.466619781s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:07.192212683Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868\") failed: Property not found. retry in 1.07016778s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:07.235078938Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae\") failed: Property not found. retry in 1.000942731s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:07.267922717Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af\") failed: Property not found. retry in 1.144370841s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:07.312578834Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47\") failed: Property not found. retry in 611.763005ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:07.470284164Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c\") failed: Property not found. retry in 789.869871ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:07.492939753Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a\") failed: Property not found. retry in 1.143239146s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:07.652713949Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878\") failed: Property not found. retry in 971.259622ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:07.668907085Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0\") failed: Property not found. retry in 1.449834867s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:07.77188179Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95\") failed: Property not found. retry in 944.256174ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:07.786007689Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759\") failed: Property not found. retry in 1.104664371s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:07.795996345Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2\") failed: Property not found. retry in 631.972354ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:07.818840867Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff\") failed: Property not found. retry in 1.171041107s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:07.962243605Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911\") failed: Property not found. retry in 1.074399316s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:08.001991185Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456\") failed: Property not found. retry in 915.487355ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:08.058655864Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47\") failed: Property not found. retry in 747.075345ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:08.095046096Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42\") failed: Property not found. retry in 1.333747256s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:08.120709018Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\") failed: Property not found. retry in 858.185521ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:08.134370219Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7\") failed: Property not found. retry in 1.361711092s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:08.176933799Z","hostname":"buskin","pid":1671,"zone":"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae\") failed: Property not found. retry in 840.91867ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:08.281890019Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab\") failed: Property not found. retry in 719.163832ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:08.292569907Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c\") failed: Property not found. retry in 630.841069ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:08.302662655Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868\") failed: Property not found. retry in 1.013436705s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:08.318296411Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2\") failed: Property not found. retry in 820.915664ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:08.461358892Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af\") failed: Property not found. retry in 840.980975ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:08.471066033Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878\") failed: Property not found. retry in 1.459070644s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:08.652107354Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846\") failed: Property not found. retry in 982.112874ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:08.669829507Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95\") failed: Property not found. retry in 1.249295869s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:08.741108663Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a\") failed: Property not found. retry in 558.453409ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:08.806980584Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47\") failed: Property not found. retry in 1.040866034s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:08.853607486Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759\") failed: Property not found. retry in 779.706695ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:08.915839453Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c\") failed: Property not found. retry in 611.395399ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:08.9548304Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456\") failed: Property not found. retry in 593.317401ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:08.998594411Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\") failed: Property not found. retry in 597.65794ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:09.018856319Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab\") failed: Property not found. retry in 1.222792892s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:09.032374227Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911\") failed: Property not found. retry in 1.166893728s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:09.094106705Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae\") failed: Property not found. retry in 678.02792ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:09.138476127Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff\") failed: Property not found. retry in 1.183591017s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:09.157467042Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0\") failed: Property not found. retry in 548.440384ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:09.241418442Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2\") failed: Property not found. retry in 1.030137829s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:09.300007394Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af\") failed: Property not found. retry in 659.088675ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:09.330022603Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868\") failed: Property not found. retry in 723.416596ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:09.356382948Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a\") failed: Property not found. retry in 883.76025ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:09.386611631Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42\") failed: Property not found. retry in 1.140852511s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:09.471827673Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7\") failed: Property not found. retry in 1.460674519s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:09.559552899Z","hostname":"buskin","pid":1671,"zone":"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c\") failed: Property not found. retry in 1.118304324s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:09.582104416Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456\") failed: Property not found. retry in 1.250709412s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:09.609301706Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\") failed: Property not found. retry in 658.924562ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:09.63395523Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846\") failed: Property not found. retry in 686.653591ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:09.666169408Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759\") failed: Property not found. retry in 1.086599709s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:09.716117378Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0\") failed: Property not found. retry in 1.357981239s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:09.802018042Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae\") failed: Property not found. retry in 1.023463499s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:09.831488392Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47\") failed: Property not found. retry in 1.291484863s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:09.913417796Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af\") failed: Property not found. retry in 1.12057639s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:10.002673519Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95\") failed: Property not found. retry in 1.199896773s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:10.014311604Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868\") failed: Property not found. retry in 909.265559ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:10.09176731Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878\") failed: Property not found. retry in 502.377739ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:10.123330106Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab\") failed: Property not found. retry in 1.293794307s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:10.271397936Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911\") failed: Property not found. retry in 1.27203808s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:10.284278853Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a\") failed: Property not found. retry in 1.022168135s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:10.298977732Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\") failed: Property not found. retry in 1.010164552s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:10.314299322Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2\") failed: Property not found. retry in 658.859268ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:10.346262495Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff\") failed: Property not found. retry in 919.02668ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:10.35960204Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846\") failed: Property not found. retry in 917.155339ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:10.372492798Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42\") failed: Property not found. retry in 1.466377351s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:10.630944613Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878\") failed: Property not found. retry in 907.112128ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:10.645014421Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c\") failed: Property not found. retry in 1.333335524s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:10.713584659Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759\") failed: Property not found. retry in 911.869929ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:10.822847105Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae\") failed: Property not found. retry in 1.480919714s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:10.867263798Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456\") failed: Property not found. retry in 556.947471ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:10.879478974Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868\") failed: Property not found. retry in 1.301139983s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:11.014564755Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2\") failed: Property not found. retry in 1.481584253s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:11.028346807Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7\") failed: Property not found. retry in 823.55567ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:11.043262881Z","hostname":"buskin","pid":1671,"zone":"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af\") failed: Property not found. retry in 863.582206ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:11.13798683Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0\") failed: Property not found. retry in 561.133913ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:11.172949556Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47\") failed: Property not found. retry in 1.395818415s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:11.215042158Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95\") failed: Property not found. retry in 1.167987699s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:11.225432821Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff\") failed: Property not found. retry in 789.127731ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:11.288678856Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846\") failed: Property not found. retry in 1.265554954s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:11.299247153Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a\") failed: Property not found. retry in 532.202189ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:11.331807266Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\") failed: Property not found. retry in 1.221370854s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:11.340391168Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456\") failed: Property not found. retry in 867.909179ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:11.45003276Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878\") failed: Property not found. retry in 1.175527203s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:11.561954072Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911\") failed: Property not found. retry in 1.214306487s","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:11.571177275Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab\") failed: Property not found. retry in 721.718023ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:11.579749276Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759\") failed: Property not found. retry in 571.528513ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:11.7450908Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring address fd00:1122:3344:101::c exists","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:11.794941459Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2002"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::c, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:11.79500107Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"Ensuring address fd00:1122:3344:101::c exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:12.368328456Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2010"} {"msg":"Zone using sled underlay as gateway","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:12.368779484Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2029"} {"msg":"Setting up Nexus service","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:15.810053859Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2073"} {"msg":"Ensuring address for OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:15.81011663Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:653"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:21.221593861Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:42607"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:21.223621837Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"a8b1943f-5d39-4ff3-8716-25b67ac36c71","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:42607","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":1640,"response_code":"200"} {"msg":"Ensuring address fd00:1122:3344:101::a exists","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:21.480681808Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2002"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::a, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:21.480747109Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"Ensuring address fd00:1122:3344:101::a exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:22.038566651Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2010"} {"msg":"Zone using sled underlay as gateway","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:22.038634852Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2029"} {"msg":"Setting up Nexus service","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:22.260324579Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2073"} {"msg":"Ensuring address for OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:22.26037659Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:653"} {"msg":"Ensuring address fd00:1122:3344:101::b exists","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:27.112814528Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2002"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::b, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:27.112879819Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"Ensuring address fd00:1122:3344:101::b exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:27.676500944Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2010"} {"msg":"Zone using sled underlay as gateway","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:27.676587905Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2029"} {"msg":"Setting up Nexus service","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:27.895592276Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2073"} {"msg":"Ensuring address for OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:27.895648716Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:653"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:34.16393458Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_external_dns_d60f3c38-1add-47dd-baca-a6103e9614ae"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:34.164015061Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_nexus_60944a67-2d69-42d2-8ce7-1215cdc06eab"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:34.164028791Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_oximeter_7fcbadc5-146c-4a16-bbe8-9fe02e4fe911"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:34.164039462Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_crucible_a493457b-acb1-4029-bcc4-1d8290fa8759"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:34.164048382Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_crucible_e17963b5-b281-4a7e-8117-ab6cd04a0a47"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:34.164071072Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_crucible_pantry_065d24ba-ce47-47d1-9175-26eabb479878"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:34.164082802Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_crucible_ab5b9d42-40cc-4445-b0f5-4f0ae4700846"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:34.164091992Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_crucible_1daa94e0-684d-4a00-9332-60a34c24e94a"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:34.164100883Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_crucible_pantry_1693d50c-fcf2-4249-86ae-6e12a08ac868"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:34.164111833Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_nexus_e72f50ee-9b20-4af9-becc-46f37972a3b0"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:34.164120683Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_crucible_bc1764f3-5054-4735-919d-fb5def7e7e42"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:34.164129693Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_crucible_pantry_fbaa903b-4740-4365-b35e-68123cc1d3a2"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:34.164140073Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_crucible_6a54d259-4fc5-4b60-a000-0fbfcf7975ff"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:34.164162294Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_crucible_fdc9e743-1329-4663-9bc9-0b2a07234456"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:34.164168314Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_crucible_6a44f502-32e2-41ae-8de2-2c1fca4407af"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:34.164173314Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_nexus_3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:34.164178644Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_clickhouse_cee74d96-bec7-4f5b-a73e-6269d90f58a7"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:34.164313356Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_external_dns_740e2473-798e-4bf1-9bf5-7a2c9ce38b95"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:34.164323047Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_crucible_3d64a4fc-53ba-4f08-9b11-25144a6b622c"} {"msg":"Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:34.16506679Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:198"} {"msg":"Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:34.16676215Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:198"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:34.168751375Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"03e4340b-5485-4d08-9499-4e350c6c12a6","remote_addr":"[fd00:1122:3344:101::1]:42546","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":58748998,"response_code":"204"} {"msg":"Finished setting up services","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:34.181356577Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1059"} {"msg":"Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:34.181731623Z","hostname":"buskin","pid":1671,"component":"RSS","file":"common/src/ledger.rs:198"} {"msg":"Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:34.182166141Z","hostname":"buskin","pid":1671,"component":"RSS","file":"common/src/ledger.rs:198"} {"msg":"Handing off control to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:34.186044059Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:549"} {"msg":"Nexus address: [fd00:1122:3344:101::b]:12221","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:34.186181132Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:551"} {"msg":"rack_network_config: RackNetworkConfigV1 {\n bgp: [],\n infra_ip_first: 192.168.1.30,\n infra_ip_last: 192.168.1.30,\n ports: [\n PortConfigV1 {\n addresses: [\n V4(\n Ipv4Network {\n addr: 10.151.1.174,\n prefix: 24,\n },\n ),\n ],\n autoneg: false,\n bgp_peers: [],\n port: \"qsfp0\",\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Network {\n addr: 0.0.0.0,\n prefix: 0,\n },\n ),\n nexthop: 10.151.1.100,\n },\n ],\n switch: Switch0,\n uplink_port_fec: None,\n uplink_port_speed: Speed40G,\n },\n ],\n rack_subnet: Ipv6Network {\n addr: fd00:1122:3344:100::,\n prefix: 56,\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:34.235703285Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/ce675376-0228-48ba-a7e8-cc3953e7c762/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:34.238398142Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/ce675376-0228-48ba-a7e8-cc3953e7c762/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:34.356775579Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/ce675376-0228-48ba-a7e8-cc3953e7c762/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:34.502130961Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8633d1af-7016-4d8b-999c-8ff9302158e4\", \"content-length\": \"124\", \"date\": \"Mon, 26 Feb 2024 07:14:34 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"8633d1af-7016-4d8b-999c-8ff9302158e4\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:34.730361163Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"634218fd-f841-4f6f-bcc5-1638614d911c\", \"content-length\": \"124\", \"date\": \"Mon, 26 Feb 2024 07:14:34 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"634218fd-f841-4f6f-bcc5-1638614d911c\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:34.85874647Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8e76b7b1-ee65-43ad-941a-3f5d965040e9\", \"content-length\": \"124\", \"date\": \"Mon, 26 Feb 2024 07:14:35 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"8e76b7b1-ee65-43ad-941a-3f5d965040e9\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:35.116612601Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"000b080f-0edb-4338-b1c9-ce65547bd034\", \"content-length\": \"124\", \"date\": \"Mon, 26 Feb 2024 07:14:34 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"000b080f-0edb-4338-b1c9-ce65547bd034\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:35.429651599Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"411a3487-54f2-43c2-8c9f-b3e73e3de6bd\", \"content-length\": \"124\", \"date\": \"Mon, 26 Feb 2024 07:14:35 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"411a3487-54f2-43c2-8c9f-b3e73e3de6bd\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:35.763833229Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d6bc39d4-8c82-4da1-8fa7-7e6140646aa6\", \"content-length\": \"124\", \"date\": \"Mon, 26 Feb 2024 07:14:36 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d6bc39d4-8c82-4da1-8fa7-7e6140646aa6\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:36.142173817Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"656d3535-a46c-4f12-895b-1db77876f2fc\", \"content-length\": \"124\", \"date\": \"Mon, 26 Feb 2024 07:14:36 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"656d3535-a46c-4f12-895b-1db77876f2fc\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:36.651602926Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0c96344c-e50a-49af-a6b8-753d9bc91973\", \"content-length\": \"124\", \"date\": \"Mon, 26 Feb 2024 07:14:36 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"0c96344c-e50a-49af-a6b8-753d9bc91973\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:37.021510636Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"060297ca-0d49-44a6-9848-e45db66d5d04\", \"content-length\": \"124\", \"date\": \"Mon, 26 Feb 2024 07:14:36 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"060297ca-0d49-44a6-9848-e45db66d5d04\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:37.514274742Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7daf9198-ced8-4090-abc1-2e6ff5244fe4\", \"content-length\": \"124\", \"date\": \"Mon, 26 Feb 2024 07:14:38 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"7daf9198-ced8-4090-abc1-2e6ff5244fe4\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:38.401363348Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"50c18cdb-0513-4494-ba0a-a2b5dd00230a\", \"content-length\": \"124\", \"date\": \"Mon, 26 Feb 2024 07:14:39 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"50c18cdb-0513-4494-ba0a-a2b5dd00230a\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:39.200524654Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"36cc6114-b17f-4598-86e5-189dc42bafd5\", \"content-length\": \"124\", \"date\": \"Mon, 26 Feb 2024 07:14:39 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"36cc6114-b17f-4598-86e5-189dc42bafd5\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:39.943098032Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"20f84919-e24a-4f65-a706-452bcf15cf07\", \"content-length\": \"124\", \"date\": \"Mon, 26 Feb 2024 07:14:41 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"20f84919-e24a-4f65-a706-452bcf15cf07\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:41.126871387Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"78027669-376a-488d-b833-5564aaa3cca4\", \"content-length\": \"124\", \"date\": \"Mon, 26 Feb 2024 07:14:42 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"78027669-376a-488d-b833-5564aaa3cca4\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:42.254421481Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6af567a6-9cb1-4a73-9449-4de39ac9fcbb\", \"content-length\": \"124\", \"date\": \"Mon, 26 Feb 2024 07:14:44 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6af567a6-9cb1-4a73-9449-4de39ac9fcbb\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:44.118508258Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"47354261-896b-4082-8a1b-1366a2ecbccd\", \"content-length\": \"124\", \"date\": \"Mon, 26 Feb 2024 07:14:46 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"47354261-896b-4082-8a1b-1366a2ecbccd\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:46.147215366Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ee714d8b-fa00-473a-ac89-6ae42bc6107d\", \"content-length\": \"124\", \"date\": \"Mon, 26 Feb 2024 07:14:49 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ee714d8b-fa00-473a-ac89-6ae42bc6107d\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:49.662300293Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:51.257727994Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:45379"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:51.257829806Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"c51e1134-016b-497a-a377-af18f28782df","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:45379","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":48,"response_code":"200"} {"msg":"Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8a2fb955-9477-404c-a524-a40f5129cc1d\", \"content-length\": \"124\", \"date\": \"Mon, 26 Feb 2024 07:14:53 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"8a2fb955-9477-404c-a524-a40f5129cc1d\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:53.340516286Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4b1eca62-780d-4ba8-a077-40f82f9d49a3\", \"content-length\": \"124\", \"date\": \"Mon, 26 Feb 2024 07:14:56 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"4b1eca62-780d-4ba8-a077-40f82f9d49a3\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:56.853025458Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: 5abda959-c9da-461c-83f2-8406a5d578e7, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:58.63348539Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"216.323808603s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"658be8f4-91ae-49e2-86f8-bad93f39daee\\\", \\\"content-length\\\": \\\"173\\\", \\\"date\\\": \\\"Mon, 26 Feb 2024 07:14:58 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: sled with id \\\\\\\"5abda959-c9da-461c-83f2-8406a5d578e7\\\\\\\"\\\", request_id: \\\"658be8f4-91ae-49e2-86f8-bad93f39daee\\\" }\""} {"msg":"failed to notify nexus about a new pool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:58.637671794Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"216.326131565s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"cf1ddeb8-c107-4ade-96af-cc1ea1c8ebd2\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Mon, 26 Feb 2024 07:14:58 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"cf1ddeb8-c107-4ade-96af-cc1ea1c8ebd2\\\" }\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: 5abda959-c9da-461c-83f2-8406a5d578e7, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:58.644855201Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"216.336941114s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"12718667-2594-4395-9793-f8e9cfdc95f9\\\", \\\"content-length\\\": \\\"173\\\", \\\"date\\\": \\\"Mon, 26 Feb 2024 07:14:58 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: sled with id \\\\\\\"5abda959-c9da-461c-83f2-8406a5d578e7\\\\\\\"\\\", request_id: \\\"12718667-2594-4395-9793-f8e9cfdc95f9\\\" }\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", sled_id: 5abda959-c9da-461c-83f2-8406a5d578e7, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:58.651148732Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"216.342633495s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"ad9d0e32-b769-44fd-8857-154c617365ce\\\", \\\"content-length\\\": \\\"173\\\", \\\"date\\\": \\\"Mon, 26 Feb 2024 07:14:58 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: sled with id \\\\\\\"5abda959-c9da-461c-83f2-8406a5d578e7\\\\\\\"\\\", request_id: \\\"ad9d0e32-b769-44fd-8857-154c617365ce\\\" }\""} {"msg":"failed to notify nexus about a new pool oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:58.65447162Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"216.343176585s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"c3536886-950a-419b-9a15-b4c6caacff05\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Mon, 26 Feb 2024 07:14:58 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"c3536886-950a-419b-9a15-b4c6caacff05\\\" }\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", sled_id: 5abda959-c9da-461c-83f2-8406a5d578e7, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:58.661173478Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"216.352092812s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"c274161b-1faf-4b1b-81a9-f9ed4d100763\\\", \\\"content-length\\\": \\\"173\\\", \\\"date\\\": \\\"Mon, 26 Feb 2024 07:14:58 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: sled with id \\\\\\\"5abda959-c9da-461c-83f2-8406a5d578e7\\\\\\\"\\\", request_id: \\\"c274161b-1faf-4b1b-81a9-f9ed4d100763\\\" }\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: 5abda959-c9da-461c-83f2-8406a5d578e7, variant: M2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:58.667578431Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"216.358840581s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"c74573a2-c4cc-43f9-b939-6ee56dc4417e\\\", \\\"content-length\\\": \\\"173\\\", \\\"date\\\": \\\"Mon, 26 Feb 2024 07:14:58 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: sled with id \\\\\\\"5abda959-c9da-461c-83f2-8406a5d578e7\\\\\\\"\\\", request_id: \\\"c74573a2-c4cc-43f9-b939-6ee56dc4417e\\\" }\""} {"msg":"failed to notify nexus about a new pool oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:58.669081358Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"216.358052216s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"9c82adff-da83-40eb-ac94-d9f0ac284271\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Mon, 26 Feb 2024 07:14:58 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"9c82adff-da83-40eb-ac94-d9f0ac284271\\\" }\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", sled_id: 5abda959-c9da-461c-83f2-8406a5d578e7, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:58.675991699Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"216.367052165s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"05f117b9-a3c6-4855-b321-107519ef967e\\\", \\\"content-length\\\": \\\"173\\\", \\\"date\\\": \\\"Mon, 26 Feb 2024 07:14:58 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: sled with id \\\\\\\"5abda959-c9da-461c-83f2-8406a5d578e7\\\\\\\"\\\", request_id: \\\"05f117b9-a3c6-4855-b321-107519ef967e\\\" }\""} {"msg":"failed to notify nexus about a new pool oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:58.678089267Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"216.366922713s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"88a5aede-ed38-45a7-97a5-3579c3b395cf\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Mon, 26 Feb 2024 07:14:58 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"88a5aede-ed38-45a7-97a5-3579c3b395cf\\\" }\""} {"msg":"failed to notify nexus about a new pool oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:58.679890068Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"216.369242374s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"c4cd685c-bed0-4249-9ce5-0c97eb210a36\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Mon, 26 Feb 2024 07:14:58 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"c4cd685c-bed0-4249-9ce5-0c97eb210a36\\\" }\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", sled_id: 5abda959-c9da-461c-83f2-8406a5d578e7, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:58.685508447Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"216.376862258s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"c0cd4af2-6fcc-4ce6-8de9-9fe48a294844\\\", \\\"content-length\\\": \\\"173\\\", \\\"date\\\": \\\"Mon, 26 Feb 2024 07:14:58 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: sled with id \\\\\\\"5abda959-c9da-461c-83f2-8406a5d578e7\\\\\\\"\\\", request_id: \\\"c0cd4af2-6fcc-4ce6-8de9-9fe48a294844\\\" }\""} {"msg":"failed to notify nexus about a new pool oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:58.687694356Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"216.377681412s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"5ae20d4b-4998-41ae-be25-91cc57682cee\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Mon, 26 Feb 2024 07:14:58 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"5ae20d4b-4998-41ae-be25-91cc57682cee\\\" }\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: 5abda959-c9da-461c-83f2-8406a5d578e7, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:58.695057995Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"216.385573482s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"f4d27bce-0f34-433c-a878-445e43ef4545\\\", \\\"content-length\\\": \\\"173\\\", \\\"date\\\": \\\"Mon, 26 Feb 2024 07:14:58 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: sled with id \\\\\\\"5abda959-c9da-461c-83f2-8406a5d578e7\\\\\\\"\\\", request_id: \\\"f4d27bce-0f34-433c-a878-445e43ef4545\\\" }\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: 5abda959-c9da-461c-83f2-8406a5d578e7, variant: M2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:58.695422032Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"216.3865878s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"48464252-ee1b-448d-b694-ab8bd6bdad0c\\\", \\\"content-length\\\": \\\"173\\\", \\\"date\\\": \\\"Mon, 26 Feb 2024 07:14:58 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: sled with id \\\\\\\"5abda959-c9da-461c-83f2-8406a5d578e7\\\\\\\"\\\", request_id: \\\"48464252-ee1b-448d-b694-ab8bd6bdad0c\\\" }\""} {"msg":"failed to notify nexus about a new pool oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:58.697258534Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"216.387431004s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"de95533f-e768-43a4-a0f9-38f28b1c4853\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Mon, 26 Feb 2024 07:14:58 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"de95533f-e768-43a4-a0f9-38f28b1c4853\\\" }\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: 5abda959-c9da-461c-83f2-8406a5d578e7, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:58.700840637Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"216.392492294s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"f4f16567-edf7-4a60-9bfb-f8344656ea3e\\\", \\\"content-length\\\": \\\"173\\\", \\\"date\\\": \\\"Mon, 26 Feb 2024 07:14:58 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: sled with id \\\\\\\"5abda959-c9da-461c-83f2-8406a5d578e7\\\\\\\"\\\", request_id: \\\"f4f16567-edf7-4a60-9bfb-f8344656ea3e\\\" }\""} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:14:58.72595468Z","hostname":"buskin","pid":1671,"component":"HardwareMonitor","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"5abda959-c9da-461c-83f2-8406a5d578e7"} {"msg":"failed to notify nexus about a new pool oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:58.729405231Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"216.417973923s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"95d2a688-ed3e-405e-a1bd-0dcbe5bdc358\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Mon, 26 Feb 2024 07:14:58 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"95d2a688-ed3e-405e-a1bd-0dcbe5bdc358\\\" }\""} {"msg":"failed to notify nexus about a new pool oxp_31bd71cd-4736-4a12-a387-9b74b050396f on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:58.73218229Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"216.421885101s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"6c5dd74c-7d7a-4452-82ec-2a5a4cb605a5\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Mon, 26 Feb 2024 07:14:58 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"6c5dd74c-7d7a-4452-82ec-2a5a4cb605a5\\\" }\""} {"msg":"failed to notify nexus about a new pool oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:58.742129795Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"216.431651774s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"a940d28d-ee95-4ba1-8e2a-8c1148e61c80\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Mon, 26 Feb 2024 07:14:58 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"a940d28d-ee95-4ba1-8e2a-8c1148e61c80\\\" }\""} {"msg":"failed to notify nexus about a new pool oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:14:58.743375937Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"216.43255943s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"4330ebd1-019f-468e-af7a-61b93e6f3b86\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Mon, 26 Feb 2024 07:14:58 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"4330ebd1-019f-468e-af7a-61b93e6f3b86\\\" }\""} {"msg":"failed to notify nexus about a new pool oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:15:00.730807548Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"218.419781636s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"50468ce0-8c29-4d26-b609-1336400ab0d0\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Mon, 26 Feb 2024 07:15:00 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"50468ce0-8c29-4d26-b609-1336400ab0d0\\\" }\""} {"msg":"failed to notify nexus about a new pool oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:15:01.070923873Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"218.75976286s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"7a3828fe-2d3d-4ddf-8972-54240397c368\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Mon, 26 Feb 2024 07:15:01 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"7a3828fe-2d3d-4ddf-8972-54240397c368\\\" }\""} {"msg":"failed to notify nexus about a new pool oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:15:01.621683461Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"219.310398505s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"5dc48bac-362e-40a4-88df-6e40c08097b0\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Mon, 26 Feb 2024 07:15:01 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"5dc48bac-362e-40a4-88df-6e40c08097b0\\\" }\""} {"msg":"failed to notify nexus about a new pool oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:15:01.682000494Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"219.371523592s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"13a14169-a619-4a0d-a7eb-fbb66205d98b\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Mon, 26 Feb 2024 07:15:00 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"13a14169-a619-4a0d-a7eb-fbb66205d98b\\\" }\""} {"msg":"failed to notify nexus about a new pool oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:15:02.017823593Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"219.707184879s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"cd4965ed-1086-468d-adc1-1c4ce2577319\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Mon, 26 Feb 2024 07:15:02 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"cd4965ed-1086-468d-adc1-1c4ce2577319\\\" }\""} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: 5abda959-c9da-461c-83f2-8406a5d578e7, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:03.363965021Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"failed to notify nexus about a new pool oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:15:03.581956293Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"221.270555344s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"eb47340f-d528-494a-9ba7-dd1886edc391\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Mon, 26 Feb 2024 07:15:03 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"eb47340f-d528-494a-9ba7-dd1886edc391\\\" }\""} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e8b201d4-316b-4663-b76f-9724ef1c80e0\", \"content-length\": \"174\", \"date\": \"Mon, 26 Feb 2024 07:15:03 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: zpool with id \\\"14b4dc87-ab46-49fb-a4b4-d361ae214c03\\\"\", request_id: \"e8b201d4-316b-4663-b76f-9724ef1c80e0\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:03.711039538Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"failed to notify nexus about a new pool oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:15:04.027120859Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"221.717141606s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"84696112-e142-4835-ad84-7c73d0f7f19c\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Mon, 26 Feb 2024 07:15:03 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"84696112-e142-4835-ad84-7c73d0f7f19c\\\" }\""} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: 5abda959-c9da-461c-83f2-8406a5d578e7, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:04.196035707Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", sled_id: 5abda959-c9da-461c-83f2-8406a5d578e7, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:04.196066767Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", sled_id: 5abda959-c9da-461c-83f2-8406a5d578e7, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:06.842388852Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: 5abda959-c9da-461c-83f2-8406a5d578e7, variant: M2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:06.842412862Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: 5abda959-c9da-461c-83f2-8406a5d578e7, variant: M2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:06.842418322Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", sled_id: 5abda959-c9da-461c-83f2-8406a5d578e7, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:06.842423072Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", sled_id: 5abda959-c9da-461c-83f2-8406a5d578e7, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:06.842427322Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: 5abda959-c9da-461c-83f2-8406a5d578e7, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:06.842431592Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: 5abda959-c9da-461c-83f2-8406a5d578e7, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:06.842435502Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: 5abda959-c9da-461c-83f2-8406a5d578e7, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:06.842439622Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:06.842444082Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"03e2c546-8f24-428a-aa3b-e4ce6e2cfdc9\", \"content-length\": \"174\", \"date\": \"Mon, 26 Feb 2024 07:15:10 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: zpool with id \\\"24b4dc87-ab46-49fb-a4b4-d361ae214c03\\\"\", request_id: \"03e2c546-8f24-428a-aa3b-e4ce6e2cfdc9\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:10.26621077Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:10.297586774Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:10.297606894Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:10.297612484Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:10.297616774Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:10.297621354Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:10.297625554Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:10.297630454Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:10.297634484Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:10.297651795Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:10.297656105Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:19.491654319Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:62206"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:19.496937783Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 864c4b26-c35e-42dc-a5b0-3e041205ccbc, kind: Service { id: d60f3c38-1add-47dd-baca-a6103e9614ae }, name: Name(\"external-dns-d60f3c38-1add-47dd-baca-a6103e9614ae\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 7])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: c9ac5e72-089e-4e66-8c75-d1c69a21aee1, kind: Service { id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95 }, name: Name(\"external-dns-740e2473-798e-4bf1-9bf5-7a2c9ce38b95\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 248])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 03f00562-72e1-4d42-8124-11bb07005618, kind: Service { id: e72f50ee-9b20-4af9-becc-46f37972a3b0 }, name: Name(\"nexus-e72f50ee-9b20-4af9-becc-46f37972a3b0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 176, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 3c85fee5-2c8b-46c2-a723-6c621ecfb3be, kind: Service { id: 3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7 }, name: Name(\"nexus-3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 215])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 850653ea-9272-4e6c-b3ae-b169328ac41c, kind: Service { id: 60944a67-2d69-42d2-8ce7-1215cdc06eab }, name: Name(\"nexus-60944a67-2d69-42d2-8ce7-1215cdc06eab\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 217, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:19.497099795Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:19.497247248Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:19.497256938Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:19.497268228Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:19.497278118Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:19.497283789Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:19.497313899Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"49363ba2-0a92-4a1a-b1a4-2caadb35a2f2","remote_addr":"[fd00:1122:3344:101::b]:62206","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":5582,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7dcdb45e-713a-45ad-92ef-a1b6a1912f4d\", \"content-length\": \"127\", \"date\": \"Mon, 26 Feb 2024 07:15:19 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"7dcdb45e-713a-45ad-92ef-a1b6a1912f4d\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:19.584529076Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:19.740859132Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:57423"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:19.742074303Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"1aca1823-22bf-4993-b234-d2a564590337","remote_addr":"[fd00:1122:3344:101::b]:57423","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":1195,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:19.742793966Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:19.743067201Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:19.745442263Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:19.74586625Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"ef5aadee-e134-4d7a-ae02-ea76fee6eabd","remote_addr":"[fd00:1122:3344:101::b]:57423","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":3607,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:21.291158137Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:47549"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:21.291255899Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"615245e8-fdc2-455e-adae-788fdef63ee2","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:47549","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":61,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:26.17779901Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:57986"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:26.177984684Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 864c4b26-c35e-42dc-a5b0-3e041205ccbc, kind: Service { id: d60f3c38-1add-47dd-baca-a6103e9614ae }, name: Name(\"external-dns-d60f3c38-1add-47dd-baca-a6103e9614ae\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 7])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: c9ac5e72-089e-4e66-8c75-d1c69a21aee1, kind: Service { id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95 }, name: Name(\"external-dns-740e2473-798e-4bf1-9bf5-7a2c9ce38b95\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 248])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 03f00562-72e1-4d42-8124-11bb07005618, kind: Service { id: e72f50ee-9b20-4af9-becc-46f37972a3b0 }, name: Name(\"nexus-e72f50ee-9b20-4af9-becc-46f37972a3b0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 176, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 3c85fee5-2c8b-46c2-a723-6c621ecfb3be, kind: Service { id: 3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7 }, name: Name(\"nexus-3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 215])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 850653ea-9272-4e6c-b3ae-b169328ac41c, kind: Service { id: 60944a67-2d69-42d2-8ce7-1215cdc06eab }, name: Name(\"nexus-60944a67-2d69-42d2-8ce7-1215cdc06eab\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 217, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:26.178050675Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:26.178089086Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:26.178096855Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:26.178106956Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:26.178306539Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:26.178314709Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:26.17835076Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"75f6ab6e-b616-4513-a1d0-e5e471815731","remote_addr":"[fd00:1122:3344:101::b]:57986","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":523,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a60119aa-c614-40bb-be68-f4a3fb228af4\", \"content-length\": \"127\", \"date\": \"Mon, 26 Feb 2024 07:15:26 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"a60119aa-c614-40bb-be68-f4a3fb228af4\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:26.237580044Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:26.322097604Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:37622"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:26.322158185Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"988d4f72-8adb-4023-9f0f-c2bd42b97b33","remote_addr":"[fd00:1122:3344:101::b]:37622","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":37,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:26.322299997Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:26.322321668Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:26.322557522Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:26.322914758Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"cea15a26-a589-4173-922e-5c3916498031","remote_addr":"[fd00:1122:3344:101::b]:37622","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":643,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:28.772878951Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::d]:38831"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:28.776498455Z","hostname":"buskin","pid":1671,"uri":"/metrics/collect/5abda959-c9da-461c-83f2-8406a5d578e7","method":"GET","req_id":"02f3a313-66f3-4c22-a117-4c2f524280b0","remote_addr":"[fd00:1122:3344:101::d]:38831","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":3596,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:32.330070171Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:44217"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:32.330238924Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 864c4b26-c35e-42dc-a5b0-3e041205ccbc, kind: Service { id: d60f3c38-1add-47dd-baca-a6103e9614ae }, name: Name(\"external-dns-d60f3c38-1add-47dd-baca-a6103e9614ae\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 7])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: c9ac5e72-089e-4e66-8c75-d1c69a21aee1, kind: Service { id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95 }, name: Name(\"external-dns-740e2473-798e-4bf1-9bf5-7a2c9ce38b95\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 248])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 03f00562-72e1-4d42-8124-11bb07005618, kind: Service { id: e72f50ee-9b20-4af9-becc-46f37972a3b0 }, name: Name(\"nexus-e72f50ee-9b20-4af9-becc-46f37972a3b0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 176, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 3c85fee5-2c8b-46c2-a723-6c621ecfb3be, kind: Service { id: 3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7 }, name: Name(\"nexus-3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 215])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 850653ea-9272-4e6c-b3ae-b169328ac41c, kind: Service { id: 60944a67-2d69-42d2-8ce7-1215cdc06eab }, name: Name(\"nexus-60944a67-2d69-42d2-8ce7-1215cdc06eab\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 217, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:32.330272174Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:32.330302575Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:32.330310095Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:32.330316165Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:32.330324475Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:32.330338576Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:32.330356376Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"06f6a642-419a-410f-9225-829f5a16d1e2","remote_addr":"[fd00:1122:3344:101::b]:44217","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":235,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4f854c10-2a79-40f4-9c12-f9dfbeec4148\", \"content-length\": \"127\", \"date\": \"Mon, 26 Feb 2024 07:15:32 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"4f854c10-2a79-40f4-9c12-f9dfbeec4148\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:32.383518513Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:32.474784842Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:47775"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:32.474830942Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"fc501c72-b34f-400a-9229-bfb624250f9a","remote_addr":"[fd00:1122:3344:101::b]:47775","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":32,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:32.474958135Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:32.474977595Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:32.475196959Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:32.475553865Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"3b9a8b29-2c83-4118-a621-e2475dc69696","remote_addr":"[fd00:1122:3344:101::b]:47775","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":614,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:45.960720656Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:43892"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:45.961149314Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 864c4b26-c35e-42dc-a5b0-3e041205ccbc, kind: Service { id: d60f3c38-1add-47dd-baca-a6103e9614ae }, name: Name(\"external-dns-d60f3c38-1add-47dd-baca-a6103e9614ae\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 7])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: c9ac5e72-089e-4e66-8c75-d1c69a21aee1, kind: Service { id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95 }, name: Name(\"external-dns-740e2473-798e-4bf1-9bf5-7a2c9ce38b95\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 248])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 03f00562-72e1-4d42-8124-11bb07005618, kind: Service { id: e72f50ee-9b20-4af9-becc-46f37972a3b0 }, name: Name(\"nexus-e72f50ee-9b20-4af9-becc-46f37972a3b0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 176, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 3c85fee5-2c8b-46c2-a723-6c621ecfb3be, kind: Service { id: 3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7 }, name: Name(\"nexus-3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 215])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 850653ea-9272-4e6c-b3ae-b169328ac41c, kind: Service { id: 60944a67-2d69-42d2-8ce7-1215cdc06eab }, name: Name(\"nexus-60944a67-2d69-42d2-8ce7-1215cdc06eab\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 217, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:45.961216055Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:45.961250106Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:45.961259546Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:45.961269926Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:45.961429579Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:45.961437339Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:45.961453579Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"15afb151-c442-4c05-b7cb-6504c39501bd","remote_addr":"[fd00:1122:3344:101::b]:43892","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":701,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7f843f5d-c69f-4ac8-9161-c211ba1f602c\", \"content-length\": \"127\", \"date\": \"Mon, 26 Feb 2024 07:15:46 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"7f843f5d-c69f-4ac8-9161-c211ba1f602c\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:46.015133095Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:46.114248552Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:61311"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:46.114321894Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"fca1db7d-895b-4e39-8afa-abd39f645e72","remote_addr":"[fd00:1122:3344:101::b]:61311","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":49,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:46.115234621Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:46.115255688Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:46.115493723Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:46.115669395Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"eb46f385-653e-4d91-907b-7e513756d9af","remote_addr":"[fd00:1122:3344:101::b]:61311","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":1269,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:51.322240637Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:63119"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:51.32242281Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"43756abc-9f1c-4c7f-9808-485568cbae38","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:63119","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":30,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:58.772470352Z","hostname":"buskin","pid":1671,"uri":"/metrics/collect/5abda959-c9da-461c-83f2-8406a5d578e7","method":"GET","req_id":"321db17d-5e3e-4344-b94a-87e43764884e","remote_addr":"[fd00:1122:3344:101::d]:38831","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":207,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:59.215948229Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:56759"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:59.216172593Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 864c4b26-c35e-42dc-a5b0-3e041205ccbc, kind: Service { id: d60f3c38-1add-47dd-baca-a6103e9614ae }, name: Name(\"external-dns-d60f3c38-1add-47dd-baca-a6103e9614ae\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 7])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: c9ac5e72-089e-4e66-8c75-d1c69a21aee1, kind: Service { id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95 }, name: Name(\"external-dns-740e2473-798e-4bf1-9bf5-7a2c9ce38b95\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 248])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 03f00562-72e1-4d42-8124-11bb07005618, kind: Service { id: e72f50ee-9b20-4af9-becc-46f37972a3b0 }, name: Name(\"nexus-e72f50ee-9b20-4af9-becc-46f37972a3b0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 176, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 3c85fee5-2c8b-46c2-a723-6c621ecfb3be, kind: Service { id: 3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7 }, name: Name(\"nexus-3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 215])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 850653ea-9272-4e6c-b3ae-b169328ac41c, kind: Service { id: 60944a67-2d69-42d2-8ce7-1215cdc06eab }, name: Name(\"nexus-60944a67-2d69-42d2-8ce7-1215cdc06eab\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 217, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:59.216235904Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:59.216269785Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:59.216278445Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:59.216290785Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:59.216301875Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:59.216310256Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:59.216348416Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"2f3469cc-6710-461d-a8f8-4cdbae3abd40","remote_addr":"[fd00:1122:3344:101::b]:56759","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":364,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0623a8ae-ca44-4363-9f3c-8255c711e170\", \"content-length\": \"127\", \"date\": \"Mon, 26 Feb 2024 07:15:59 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"0623a8ae-ca44-4363-9f3c-8255c711e170\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:59.266766255Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:59.36577713Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:41384"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:59.365835751Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"61ce8e4e-a4b1-48da-b49e-19121a9614af","remote_addr":"[fd00:1122:3344:101::b]:41384","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":41,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:59.366643295Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:59.366686866Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:59.366963241Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:15:59.367126694Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"0e4a8522-5fe1-4f8e-93cd-6e7e930af2b3","remote_addr":"[fd00:1122:3344:101::b]:41384","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":1157,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:16:08.818198976Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:37111"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:16:08.818382529Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 864c4b26-c35e-42dc-a5b0-3e041205ccbc, kind: Service { id: d60f3c38-1add-47dd-baca-a6103e9614ae }, name: Name(\"external-dns-d60f3c38-1add-47dd-baca-a6103e9614ae\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 7])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: c9ac5e72-089e-4e66-8c75-d1c69a21aee1, kind: Service { id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95 }, name: Name(\"external-dns-740e2473-798e-4bf1-9bf5-7a2c9ce38b95\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 248])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 03f00562-72e1-4d42-8124-11bb07005618, kind: Service { id: e72f50ee-9b20-4af9-becc-46f37972a3b0 }, name: Name(\"nexus-e72f50ee-9b20-4af9-becc-46f37972a3b0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 176, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 3c85fee5-2c8b-46c2-a723-6c621ecfb3be, kind: Service { id: 3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7 }, name: Name(\"nexus-3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 215])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 850653ea-9272-4e6c-b3ae-b169328ac41c, kind: Service { id: 60944a67-2d69-42d2-8ce7-1215cdc06eab }, name: Name(\"nexus-60944a67-2d69-42d2-8ce7-1215cdc06eab\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 217, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:16:08.81841541Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:16:08.81843623Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:16:08.81844357Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:16:08.81845096Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:16:08.81845773Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:16:08.818463921Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:16:08.818477321Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"1f1ffc4d-86dc-44db-ab7f-8d8014aee125","remote_addr":"[fd00:1122:3344:101::b]:37111","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":216,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"cc01a117-3569-47be-8633-fdff58290791\", \"content-length\": \"127\", \"date\": \"Mon, 26 Feb 2024 07:16:08 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"cc01a117-3569-47be-8633-fdff58290791\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:16:08.872161217Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:16:08.98581765Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:49617"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:16:08.985881781Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"e25a883d-9ec6-4a30-b22b-f2f34285906d","remote_addr":"[fd00:1122:3344:101::b]:49617","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":43,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:16:08.986772807Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:16:08.986793548Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:16:08.987137794Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:16:08.987321077Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"1f1a38de-ef21-4b27-9578-5d5998f27b3f","remote_addr":"[fd00:1122:3344:101::b]:49617","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":1323,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:16:21.354730151Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:33940"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:16:21.354810613Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"12ae491f-071c-4cdd-bae2-7349d8f97da1","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:33940","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":44,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:16:28.773473522Z","hostname":"buskin","pid":1671,"uri":"/metrics/collect/5abda959-c9da-461c-83f2-8406a5d578e7","method":"GET","req_id":"54ba9e6c-7e4f-4958-a443-10f6e1c4e698","remote_addr":"[fd00:1122:3344:101::d]:38831","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":644,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:16:30.420328259Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:53366"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:16:30.420492062Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 864c4b26-c35e-42dc-a5b0-3e041205ccbc, kind: Service { id: d60f3c38-1add-47dd-baca-a6103e9614ae }, name: Name(\"external-dns-d60f3c38-1add-47dd-baca-a6103e9614ae\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 7])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: c9ac5e72-089e-4e66-8c75-d1c69a21aee1, kind: Service { id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95 }, name: Name(\"external-dns-740e2473-798e-4bf1-9bf5-7a2c9ce38b95\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 248])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 03f00562-72e1-4d42-8124-11bb07005618, kind: Service { id: e72f50ee-9b20-4af9-becc-46f37972a3b0 }, name: Name(\"nexus-e72f50ee-9b20-4af9-becc-46f37972a3b0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 176, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 3c85fee5-2c8b-46c2-a723-6c621ecfb3be, kind: Service { id: 3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7 }, name: Name(\"nexus-3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 215])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 850653ea-9272-4e6c-b3ae-b169328ac41c, kind: Service { id: 60944a67-2d69-42d2-8ce7-1215cdc06eab }, name: Name(\"nexus-60944a67-2d69-42d2-8ce7-1215cdc06eab\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 217, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:16:30.420531613Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:16:30.420556793Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:16:30.420564354Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:16:30.420571494Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:16:30.420577384Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:16:30.420586804Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:16:30.420596944Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"3db348bc-c9b8-4eb9-b2aa-3046f687bdb5","remote_addr":"[fd00:1122:3344:101::b]:53366","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":224,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"259b8244-bc6b-48f9-9aec-8e6b52355c8c\", \"content-length\": \"127\", \"date\": \"Mon, 26 Feb 2024 07:16:30 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"259b8244-bc6b-48f9-9aec-8e6b52355c8c\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:16:30.477900224Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:16:30.57982194Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:60630"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:16:30.579902531Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"0db8e5ec-f517-45a0-b6e2-ab98b00c9702","remote_addr":"[fd00:1122:3344:101::b]:60630","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":51,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:16:30.580080665Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:16:30.580116905Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:16:30.58037143Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:16:30.580551153Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"21213f6d-c66a-447f-9bae-9a182a9974b7","remote_addr":"[fd00:1122:3344:101::b]:60630","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":496,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:16:51.388304645Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:56349"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:16:51.388355246Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"d12d3836-d469-49b3-8496-033ea1bb5c86","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:56349","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":36,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:16:58.773656847Z","hostname":"buskin","pid":1671,"uri":"/metrics/collect/5abda959-c9da-461c-83f2-8406a5d578e7","method":"GET","req_id":"22d4b777-f59b-41a2-904f-4f9646306f0d","remote_addr":"[fd00:1122:3344:101::d]:38831","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":189,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:17:04.428803754Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:37867"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:17:04.429135249Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 864c4b26-c35e-42dc-a5b0-3e041205ccbc, kind: Service { id: d60f3c38-1add-47dd-baca-a6103e9614ae }, name: Name(\"external-dns-d60f3c38-1add-47dd-baca-a6103e9614ae\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 7])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: c9ac5e72-089e-4e66-8c75-d1c69a21aee1, kind: Service { id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95 }, name: Name(\"external-dns-740e2473-798e-4bf1-9bf5-7a2c9ce38b95\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 248])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 03f00562-72e1-4d42-8124-11bb07005618, kind: Service { id: e72f50ee-9b20-4af9-becc-46f37972a3b0 }, name: Name(\"nexus-e72f50ee-9b20-4af9-becc-46f37972a3b0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 176, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 3c85fee5-2c8b-46c2-a723-6c621ecfb3be, kind: Service { id: 3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7 }, name: Name(\"nexus-3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 215])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 850653ea-9272-4e6c-b3ae-b169328ac41c, kind: Service { id: 60944a67-2d69-42d2-8ce7-1215cdc06eab }, name: Name(\"nexus-60944a67-2d69-42d2-8ce7-1215cdc06eab\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 217, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:17:04.429195201Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:17:04.429229311Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:17:04.429238001Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:17:04.429250082Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:17:04.429271142Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:17:04.429278192Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:17:04.429311763Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"4c57ecf3-3701-4b48-b18e-70233e12cf2e","remote_addr":"[fd00:1122:3344:101::b]:37867","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":408,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b93f319d-5ef2-41aa-8f6d-99ff5eff6241\", \"content-length\": \"127\", \"date\": \"Mon, 26 Feb 2024 07:17:04 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"b93f319d-5ef2-41aa-8f6d-99ff5eff6241\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:17:04.483906395Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:17:04.579572301Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:38058"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:17:04.579665083Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"1f8909b6-74d8-43d0-b1ee-caffe3c4048f","remote_addr":"[fd00:1122:3344:101::b]:38058","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":57,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:17:04.579868476Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:17:04.579900977Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:17:04.580227253Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:17:04.580501797Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"d6d5eb53-7bbc-420a-9ed5-fc0ca2c32b9f","remote_addr":"[fd00:1122:3344:101::b]:38058","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":661,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:17:21.420869802Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:59906"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:17:21.420915233Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"c0678c08-57f4-44eb-b32f-8455456337d7","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:59906","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":32,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:17:28.717738845Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:40544"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:17:28.717912638Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 864c4b26-c35e-42dc-a5b0-3e041205ccbc, kind: Service { id: d60f3c38-1add-47dd-baca-a6103e9614ae }, name: Name(\"external-dns-d60f3c38-1add-47dd-baca-a6103e9614ae\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 7])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: c9ac5e72-089e-4e66-8c75-d1c69a21aee1, kind: Service { id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95 }, name: Name(\"external-dns-740e2473-798e-4bf1-9bf5-7a2c9ce38b95\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 248])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 03f00562-72e1-4d42-8124-11bb07005618, kind: Service { id: e72f50ee-9b20-4af9-becc-46f37972a3b0 }, name: Name(\"nexus-e72f50ee-9b20-4af9-becc-46f37972a3b0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 176, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 3c85fee5-2c8b-46c2-a723-6c621ecfb3be, kind: Service { id: 3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7 }, name: Name(\"nexus-3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 215])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 850653ea-9272-4e6c-b3ae-b169328ac41c, kind: Service { id: 60944a67-2d69-42d2-8ce7-1215cdc06eab }, name: Name(\"nexus-60944a67-2d69-42d2-8ce7-1215cdc06eab\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 217, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:17:28.717951828Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:17:28.717982209Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:17:28.717990599Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:17:28.717997479Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:17:28.718003969Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:17:28.71801009Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:17:28.71802835Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"ae294a3c-318f-429c-9a40-65794885d3e2","remote_addr":"[fd00:1122:3344:101::b]:40544","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":252,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:17:28.774348983Z","hostname":"buskin","pid":1671,"uri":"/metrics/collect/5abda959-c9da-461c-83f2-8406a5d578e7","method":"GET","req_id":"88e0d310-e45e-4840-ad7c-18089970af66","remote_addr":"[fd00:1122:3344:101::d]:38831","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":178,"response_code":"200"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7a2a0adc-6bad-4839-a2f6-02cf0755851b\", \"content-length\": \"127\", \"date\": \"Mon, 26 Feb 2024 07:17:28 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"7a2a0adc-6bad-4839-a2f6-02cf0755851b\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:17:28.775808298Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:17:28.876678916Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:44476"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:17:28.877200875Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"fc77f750-9ff5-46ef-8169-bf4058d728b2","remote_addr":"[fd00:1122:3344:101::b]:44476","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":496,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:17:28.877358108Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:17:28.877377768Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:17:28.877661973Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:17:28.877856817Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"ee9472d6-22af-41d6-a46e-8e0def61f0e4","remote_addr":"[fd00:1122:3344:101::b]:44476","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":518,"response_code":"200"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:17:39.24579365Z","hostname":"buskin","pid":1671,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:776"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:17:51.452994391Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:46494"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:17:51.453040012Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"42aa2517-abc6-433f-af6a-daf7043fb615","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:46494","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":32,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:17:58.775537776Z","hostname":"buskin","pid":1671,"uri":"/metrics/collect/5abda959-c9da-461c-83f2-8406a5d578e7","method":"GET","req_id":"8248f92c-0b48-41bb-9b81-6888e897b86e","remote_addr":"[fd00:1122:3344:101::d]:38831","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":777,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:18:16.174750611Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:36430"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:18:16.174937755Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 864c4b26-c35e-42dc-a5b0-3e041205ccbc, kind: Service { id: d60f3c38-1add-47dd-baca-a6103e9614ae }, name: Name(\"external-dns-d60f3c38-1add-47dd-baca-a6103e9614ae\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 7])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: c9ac5e72-089e-4e66-8c75-d1c69a21aee1, kind: Service { id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95 }, name: Name(\"external-dns-740e2473-798e-4bf1-9bf5-7a2c9ce38b95\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 248])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 03f00562-72e1-4d42-8124-11bb07005618, kind: Service { id: e72f50ee-9b20-4af9-becc-46f37972a3b0 }, name: Name(\"nexus-e72f50ee-9b20-4af9-becc-46f37972a3b0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 176, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 3c85fee5-2c8b-46c2-a723-6c621ecfb3be, kind: Service { id: 3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7 }, name: Name(\"nexus-3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 215])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 850653ea-9272-4e6c-b3ae-b169328ac41c, kind: Service { id: 60944a67-2d69-42d2-8ce7-1215cdc06eab }, name: Name(\"nexus-60944a67-2d69-42d2-8ce7-1215cdc06eab\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 217, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:18:16.174977595Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:18:16.175000245Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:18:16.175005056Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:18:16.175009636Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:18:16.175018106Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:18:16.175022736Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:18:16.175048226Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"d59d8b87-eaf7-488f-beb9-4a4a71965a51","remote_addr":"[fd00:1122:3344:101::b]:36430","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":240,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0e73d366-794a-4d01-bdf8-271eb7bb333c\", \"content-length\": \"127\", \"date\": \"Mon, 26 Feb 2024 07:18:16 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"0e73d366-794a-4d01-bdf8-271eb7bb333c\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:18:16.243731277Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:18:16.375645052Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:51315"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:18:16.375713493Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"6c10b15f-be4d-429f-8017-f3b77eca813d","remote_addr":"[fd00:1122:3344:101::b]:51315","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":46,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:18:16.375881496Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:18:16.375909367Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:18:16.376157561Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:18:16.377191149Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"82aecd83-3179-4434-b69d-62639b372156","remote_addr":"[fd00:1122:3344:101::b]:51315","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":1330,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:18:21.485443906Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:39113"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:18:21.486622647Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"3cecd493-7f21-4a9e-9e00-b49ead59cb8e","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:39113","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":529,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:18:28.776300113Z","hostname":"buskin","pid":1671,"uri":"/metrics/collect/5abda959-c9da-461c-83f2-8406a5d578e7","method":"GET","req_id":"f69d3669-341c-46c9-a0aa-5fa573f66974","remote_addr":"[fd00:1122:3344:101::d]:38831","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":1392,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:18:51.52060079Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:36036"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:18:51.521135539Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"9c5d383f-2136-419f-b456-313f010f886c","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:36036","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":505,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:18:58.775512653Z","hostname":"buskin","pid":1671,"uri":"/metrics/collect/5abda959-c9da-461c-83f2-8406a5d578e7","method":"GET","req_id":"40e06721-db02-4c29-b2aa-852c56d70ddf","remote_addr":"[fd00:1122:3344:101::d]:38831","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":189,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:19:08.234864861Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:37817"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:19:08.235024524Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 864c4b26-c35e-42dc-a5b0-3e041205ccbc, kind: Service { id: d60f3c38-1add-47dd-baca-a6103e9614ae }, name: Name(\"external-dns-d60f3c38-1add-47dd-baca-a6103e9614ae\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 7])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: c9ac5e72-089e-4e66-8c75-d1c69a21aee1, kind: Service { id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95 }, name: Name(\"external-dns-740e2473-798e-4bf1-9bf5-7a2c9ce38b95\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 248])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 03f00562-72e1-4d42-8124-11bb07005618, kind: Service { id: e72f50ee-9b20-4af9-becc-46f37972a3b0 }, name: Name(\"nexus-e72f50ee-9b20-4af9-becc-46f37972a3b0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 176, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 3c85fee5-2c8b-46c2-a723-6c621ecfb3be, kind: Service { id: 3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7 }, name: Name(\"nexus-3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 215])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 850653ea-9272-4e6c-b3ae-b169328ac41c, kind: Service { id: 60944a67-2d69-42d2-8ce7-1215cdc06eab }, name: Name(\"nexus-60944a67-2d69-42d2-8ce7-1215cdc06eab\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 217, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:19:08.235073305Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:19:08.235091525Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:19:08.235099756Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:19:08.235113656Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:19:08.235121956Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:19:08.235127586Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:19:08.235156426Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"81a3c382-58f1-485c-8fad-f1ad2e3d6a8b","remote_addr":"[fd00:1122:3344:101::b]:37817","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":263,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"41a0bfc6-5ec4-46b8-8a8f-10cc1845d70c\", \"content-length\": \"127\", \"date\": \"Mon, 26 Feb 2024 07:19:08 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"41a0bfc6-5ec4-46b8-8a8f-10cc1845d70c\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:19:08.300666111Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:19:08.423878653Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:63356"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:19:08.424548385Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"497afd92-297e-4840-b531-b332e58875d8","remote_addr":"[fd00:1122:3344:101::b]:63356","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":648,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:19:08.424746348Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:19:08.424777049Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:19:08.425513402Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:19:08.426192094Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"42367573-566f-481c-b084-8427ca11ee42","remote_addr":"[fd00:1122:3344:101::b]:63356","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":1467,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:19:21.552962473Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:54940"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:19:21.553654095Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"8b9c134d-8c73-4c10-8d12-750d76390793","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:54940","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":669,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:19:28.775416035Z","hostname":"buskin","pid":1671,"uri":"/metrics/collect/5abda959-c9da-461c-83f2-8406a5d578e7","method":"GET","req_id":"d3b3195d-39cf-4c0b-8362-2e91f8a07401","remote_addr":"[fd00:1122:3344:101::d]:38831","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":212,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:19:42.719478419Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:40247"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:19:42.719621532Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 864c4b26-c35e-42dc-a5b0-3e041205ccbc, kind: Service { id: d60f3c38-1add-47dd-baca-a6103e9614ae }, name: Name(\"external-dns-d60f3c38-1add-47dd-baca-a6103e9614ae\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 7])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: c9ac5e72-089e-4e66-8c75-d1c69a21aee1, kind: Service { id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95 }, name: Name(\"external-dns-740e2473-798e-4bf1-9bf5-7a2c9ce38b95\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 248])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 03f00562-72e1-4d42-8124-11bb07005618, kind: Service { id: e72f50ee-9b20-4af9-becc-46f37972a3b0 }, name: Name(\"nexus-e72f50ee-9b20-4af9-becc-46f37972a3b0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 176, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 3c85fee5-2c8b-46c2-a723-6c621ecfb3be, kind: Service { id: 3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7 }, name: Name(\"nexus-3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 215])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 850653ea-9272-4e6c-b3ae-b169328ac41c, kind: Service { id: 60944a67-2d69-42d2-8ce7-1215cdc06eab }, name: Name(\"nexus-60944a67-2d69-42d2-8ce7-1215cdc06eab\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 217, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:19:42.719654033Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:19:42.719680433Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:19:42.719685863Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:19:42.719691243Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:19:42.719696503Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:19:42.719701194Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:19:42.719724164Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"2155b3e0-9ab1-4e20-8cb6-359f976f9615","remote_addr":"[fd00:1122:3344:101::b]:40247","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":214,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"13c06602-6aa8-40e5-8018-4d616d3402b2\", \"content-length\": \"127\", \"date\": \"Mon, 26 Feb 2024 07:19:42 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"13c06602-6aa8-40e5-8018-4d616d3402b2\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:19:42.775876324Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:19:42.87891749Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:34111"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:19:42.878963431Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"811b5973-2669-48b7-85b5-4f64ecd97c10","remote_addr":"[fd00:1122:3344:101::b]:34111","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":34,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:19:42.879170284Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:19:42.879189325Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:19:42.879415109Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:19:42.879594302Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"0cc73895-4f85-45b2-95f1-83b191075288","remote_addr":"[fd00:1122:3344:101::b]:34111","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":446,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:19:51.58889123Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:55482"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:19:51.588964601Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"a5b91d9e-b842-4946-8819-b5715ce01bad","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:55482","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":45,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:19:58.775856966Z","hostname":"buskin","pid":1671,"uri":"/metrics/collect/5abda959-c9da-461c-83f2-8406a5d578e7","method":"GET","req_id":"b55a8926-0a81-4dc4-bf02-baafc577cda7","remote_addr":"[fd00:1122:3344:101::d]:38831","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":188,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:20:21.621484808Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:57972"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:20:21.621532649Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"ef2d0f19-4730-4a55-aff4-876606875b13","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:57972","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":34,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:20:28.776365709Z","hostname":"buskin","pid":1671,"uri":"/metrics/collect/5abda959-c9da-461c-83f2-8406a5d578e7","method":"GET","req_id":"43dadc69-ab64-4d35-b915-b18645589fef","remote_addr":"[fd00:1122:3344:101::d]:38831","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":204,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:20:36.828379732Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:61345"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:20:36.828596916Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 864c4b26-c35e-42dc-a5b0-3e041205ccbc, kind: Service { id: d60f3c38-1add-47dd-baca-a6103e9614ae }, name: Name(\"external-dns-d60f3c38-1add-47dd-baca-a6103e9614ae\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 7])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: c9ac5e72-089e-4e66-8c75-d1c69a21aee1, kind: Service { id: 740e2473-798e-4bf1-9bf5-7a2c9ce38b95 }, name: Name(\"external-dns-740e2473-798e-4bf1-9bf5-7a2c9ce38b95\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 248])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 03f00562-72e1-4d42-8124-11bb07005618, kind: Service { id: e72f50ee-9b20-4af9-becc-46f37972a3b0 }, name: Name(\"nexus-e72f50ee-9b20-4af9-becc-46f37972a3b0\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 176, 228])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 3c85fee5-2c8b-46c2-a723-6c621ecfb3be, kind: Service { id: 3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7 }, name: Name(\"nexus-3a6ee1c4-7319-4fc1-8ff2-4b8ad7f7cdb7\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 215])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 850653ea-9272-4e6c-b3ae-b169328ac41c, kind: Service { id: 60944a67-2d69-42d2-8ce7-1215cdc06eab }, name: Name(\"nexus-60944a67-2d69-42d2-8ce7-1215cdc06eab\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 217, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:20:36.828660587Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:20:36.828691717Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:20:36.828699758Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:20:36.828707198Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:20:36.828717318Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:20:36.828726308Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:20:36.828766519Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"61bfd563-caba-47f6-86c4-94e474d66afe","remote_addr":"[fd00:1122:3344:101::b]:61345","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":347,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5058e458-eb55-4034-a19e-5a2e4131003c\", \"content-length\": \"127\", \"date\": \"Mon, 26 Feb 2024 07:20:36 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"5058e458-eb55-4034-a19e-5a2e4131003c\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:20:36.881770683Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:20:36.976505333Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:52918"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:20:36.976563034Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"5f602eeb-cc7f-40f1-99f9-3ad97904fdad","remote_addr":"[fd00:1122:3344:101::b]:52918","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":39,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:20:36.976699943Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:20:36.9767219Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:20:36.976950326Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:20:36.977127216Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"c1e70d0b-bbb3-435c-accf-07c03e415c46","remote_addr":"[fd00:1122:3344:101::b]:52918","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":477,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:20:51.65312306Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:46053"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:20:51.653183851Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"9c25c9fd-e9a7-4c05-8086-79f7215f96fa","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:46053","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":37,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:20:57.656471243Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:61870"} {"msg":"ensuring instance is registered","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:20:57.6676399Z","hostname":"buskin","pid":1671,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:586","propolis_addr":"[fd00:1122:3344:101::1:0]:12400","vmm_runtime":"VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-02-26T07:20:57.342847Z }","instance_runtime":"InstanceRuntimeState { propolis_id: Some(8c58d64a-5686-44c4-ace6-adbb0c8c7347), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-02-26T07:20:57.366781248Z }","hardware":"InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: 032caeae-c9ce-4568-b322-19d7207fd125, kind: Instance { id: 8c1510f5-7019-4371-ba38-03f670b781e9 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 242, 36, 60])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(3501953), primary: true, slot: 0 }], source_nat: SourceNatConfig { ip: 10.151.1.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.1.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 032caeae-c9ce-4568-b322-19d7207fd125, kind: Instance { id: 8c1510f5-7019-4371-ba38-03f670b781e9 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 242, 36, 60])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(3501953), primary: true, slot: 0 }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 032caeae-c9ce-4568-b322-19d7207fd125, kind: Instance { id: 8c1510f5-7019-4371-ba38-03f670b781e9 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 242, 36, 60])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(3501953), primary: true, slot: 0 }], filter_hosts: Some([Vpc(Vni(3501953))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 032caeae-c9ce-4568-b322-19d7207fd125, kind: Instance { id: 8c1510f5-7019-4371-ba38-03f670b781e9 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 242, 36, 60])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(3501953), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-ecef3bc7e8\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 1db0dc18-2ff4-49ff-854b-668e7d436571, read_only_parent: Some(Volume { block_size: 512, id: 4a8ab656-54f6-43d3-9e93-f8956670967c, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 9438d2b2-b6ea-40a1-b98e-b23df28731d3, key: Some(\"c0Mfx7PCuxa0pnEwqQ71E/9uWhHQUVoQtA66zhe38eM=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::1a]:19002\", \"[fd00:1122:3344:101::16]:19001\", \"[fd00:1122:3344:101::14]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 1db0dc18-2ff4-49ff-854b-668e7d436571, key: Some(\"YRoHGYeAoySO1pD8lI2b9DpNtmNUw50qHwIVO2CT2Xo=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::18]:19000\", \"[fd00:1122:3344:101::14]:19002\", \"[fd00:1122:3344:101::1a]:19003\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAAChnDpaWFpYAACcOlpYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAoZw6WlhaWAAAnDpaWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiI4YzE1MTBmNS03MDE5LTQzNzEtYmEzOC0wM2Y2NzBiNzgxZTkiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJUFBKNEVtMU9HQ2dldUp1U1dZS1pNU3NybVo0OW5hdlVya09DUFRrc0lROCJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }","propolis_id":"8c58d64a-5686-44c4-ace6-adbb0c8c7347","instance_id":"8c1510f5-7019-4371-ba38-03f670b781e9"} {"msg":"registering new instance","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:20:57.667747802Z","hostname":"buskin","pid":1671,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:619","instance_id":"8c1510f5-7019-4371-ba38-03f670b781e9"} {"msg":"initializing new Instance","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:20:57.6699058Z","hostname":"buskin","pid":1671,"component":"InstanceManager","file":"sled-agent/src/instance.rs:932","state":"InstanceInitialState { hardware: InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: 032caeae-c9ce-4568-b322-19d7207fd125, kind: Instance { id: 8c1510f5-7019-4371-ba38-03f670b781e9 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 242, 36, 60])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(3501953), primary: true, slot: 0 }], source_nat: SourceNatConfig { ip: 10.151.1.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.1.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 032caeae-c9ce-4568-b322-19d7207fd125, kind: Instance { id: 8c1510f5-7019-4371-ba38-03f670b781e9 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 242, 36, 60])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(3501953), primary: true, slot: 0 }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 032caeae-c9ce-4568-b322-19d7207fd125, kind: Instance { id: 8c1510f5-7019-4371-ba38-03f670b781e9 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 242, 36, 60])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(3501953), primary: true, slot: 0 }], filter_hosts: Some([Vpc(Vni(3501953))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 032caeae-c9ce-4568-b322-19d7207fd125, kind: Instance { id: 8c1510f5-7019-4371-ba38-03f670b781e9 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 242, 36, 60])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(3501953), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-ecef3bc7e8\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 1db0dc18-2ff4-49ff-854b-668e7d436571, read_only_parent: Some(Volume { block_size: 512, id: 4a8ab656-54f6-43d3-9e93-f8956670967c, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 9438d2b2-b6ea-40a1-b98e-b23df28731d3, key: Some(\"c0Mfx7PCuxa0pnEwqQ71E/9uWhHQUVoQtA66zhe38eM=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::1a]:19002\", \"[fd00:1122:3344:101::16]:19001\", \"[fd00:1122:3344:101::14]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 1db0dc18-2ff4-49ff-854b-668e7d436571, key: Some(\"YRoHGYeAoySO1pD8lI2b9DpNtmNUw50qHwIVO2CT2Xo=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::18]:19000\", \"[fd00:1122:3344:101::14]:19002\", \"[fd00:1122:3344:101::1a]:19003\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAAChnDpaWFpYAACcOlpYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAoZw6WlhaWAAAnDpaWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiI4YzE1MTBmNS03MDE5LTQzNzEtYmEzOC0wM2Y2NzBiNzgxZTkiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJUFBKNEVtMU9HQ2dldUp1U1dZS1pNU3NybVo0OW5hdlVya09DUFRrc0lROCJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }, instance_runtime: InstanceRuntimeState { propolis_id: Some(8c58d64a-5686-44c4-ace6-adbb0c8c7347), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-02-26T07:20:57.366781248Z }, vmm_runtime: VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-02-26T07:20:57.342847Z }, propolis_addr: [fd00:1122:3344:101::1:0]:12400 }","propolis_id":"8c58d64a-5686-44c4-ace6-adbb0c8c7347","instance_id":"8c1510f5-7019-4371-ba38-03f670b781e9"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:20:57.671515939Z","hostname":"buskin","pid":1671,"uri":"/instances/8c1510f5-7019-4371-ba38-03f670b781e9","method":"PUT","req_id":"13198cb8-7c70-4aae-b85e-c6922cef2666","remote_addr":"[fd00:1122:3344:101::a]:61870","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":14993,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:20:57.763849566Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:44490"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:20:57.784826266Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte6\", ip: 172.30.0.5, mac: MacAddr6([168, 64, 37, 242, 36, 60]), slot: 0, vni: Vni { inner: 3501953 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.0.1 }, vnic: \"vopte6\" } }"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:20:57.785504908Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Configuring new Omicron zone: oxz_propolis-server_8c58d64a-5686-44c4-ace6-adbb0c8c7347","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:20:57.81739229Z","hostname":"buskin","pid":1671,"instance_id":"8c1510f5-7019-4371-ba38-03f670b781e9","component":"InstanceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_propolis-server_8c58d64a-5686-44c4-ace6-adbb0c8c7347","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:20:57.837406243Z","hostname":"buskin","pid":1671,"instance_id":"8c1510f5-7019-4371-ba38-03f670b781e9","component":"InstanceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:20:58.777006524Z","hostname":"buskin","pid":1671,"uri":"/metrics/collect/5abda959-c9da-461c-83f2-8406a5d578e7","method":"GET","req_id":"e9844364-788f-4773-8664-9bf17483c8d4","remote_addr":"[fd00:1122:3344:101::d]:38831","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":586,"response_code":"200"} {"msg":"Profile for oxz_propolis-server_8c58d64a-5686-44c4-ace6-adbb0c8c7347:\n\n\n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:20:59.50580744Z","hostname":"buskin","pid":1671,"instance_id":"8c1510f5-7019-4371-ba38-03f670b781e9","component":"InstanceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:20:59.506484402Z","hostname":"buskin","pid":1671,"zone":"oxz_propolis-server_8c58d64a-5686-44c4-ace6-adbb0c8c7347","instance_id":"8c1510f5-7019-4371-ba38-03f670b781e9","component":"InstanceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_propolis-server_8c58d64a-5686-44c4-ace6-adbb0c8c7347\") failed: Property not found. retry in 70.650005ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:20:59.700379439Z","hostname":"buskin","pid":1671,"zone":"oxz_propolis-server_8c58d64a-5686-44c4-ace6-adbb0c8c7347","instance_id":"8c1510f5-7019-4371-ba38-03f670b781e9","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_8c58d64a-5686-44c4-ace6-adbb0c8c7347\") failed: Property not found. retry in 94.851399ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:20:59.794118221Z","hostname":"buskin","pid":1671,"zone":"oxz_propolis-server_8c58d64a-5686-44c4-ace6-adbb0c8c7347","instance_id":"8c1510f5-7019-4371-ba38-03f670b781e9","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_8c58d64a-5686-44c4-ace6-adbb0c8c7347\") failed: Property not found. retry in 109.552555ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:20:59.901601486Z","hostname":"buskin","pid":1671,"zone":"oxz_propolis-server_8c58d64a-5686-44c4-ace6-adbb0c8c7347","instance_id":"8c1510f5-7019-4371-ba38-03f670b781e9","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_8c58d64a-5686-44c4-ace6-adbb0c8c7347\") failed: Property not found. retry in 435.434855ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:21:00.024818538Z","hostname":"buskin","pid":1671,"zone":"oxz_propolis-server_8c58d64a-5686-44c4-ace6-adbb0c8c7347","instance_id":"8c1510f5-7019-4371-ba38-03f670b781e9","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_8c58d64a-5686-44c4-ace6-adbb0c8c7347\") failed: Property not found. retry in 524.762388ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:21:00.471655554Z","hostname":"buskin","pid":1671,"zone":"oxz_propolis-server_8c58d64a-5686-44c4-ace6-adbb0c8c7347","instance_id":"8c1510f5-7019-4371-ba38-03f670b781e9","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_8c58d64a-5686-44c4-ace6-adbb0c8c7347\") failed: Property not found. retry in 566.543386ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:21:01.010263747Z","hostname":"buskin","pid":1671,"zone":"oxz_propolis-server_8c58d64a-5686-44c4-ace6-adbb0c8c7347","instance_id":"8c1510f5-7019-4371-ba38-03f670b781e9","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_8c58d64a-5686-44c4-ace6-adbb0c8c7347\") failed: Property not found. retry in 738.918782ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:21:01.586824239Z","hostname":"buskin","pid":1671,"zone":"oxz_propolis-server_8c58d64a-5686-44c4-ace6-adbb0c8c7347","instance_id":"8c1510f5-7019-4371-ba38-03f670b781e9","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_8c58d64a-5686-44c4-ace6-adbb0c8c7347\") failed: Property not found. retry in 911.562557ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:21:02.335963763Z","hostname":"buskin","pid":1671,"zone":"oxz_propolis-server_8c58d64a-5686-44c4-ace6-adbb0c8c7347","instance_id":"8c1510f5-7019-4371-ba38-03f670b781e9","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_8c58d64a-5686-44c4-ace6-adbb0c8c7347\") failed: Property not found. retry in 773.851148ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:21:03.25830692Z","hostname":"buskin","pid":1671,"zone":"oxz_propolis-server_8c58d64a-5686-44c4-ace6-adbb0c8c7347","instance_id":"8c1510f5-7019-4371-ba38-03f670b781e9","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_8c58d64a-5686-44c4-ace6-adbb0c8c7347\") failed: Property not found. retry in 850.795909ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:21:04.042795978Z","hostname":"buskin","pid":1671,"zone":"oxz_propolis-server_8c58d64a-5686-44c4-ace6-adbb0c8c7347","instance_id":"8c1510f5-7019-4371-ba38-03f670b781e9","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Started propolis in zone: oxz_propolis-server_8c58d64a-5686-44c4-ace6-adbb0c8c7347","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:21:04.92178195Z","hostname":"buskin","pid":1671,"instance_id":"8c1510f5-7019-4371-ba38-03f670b781e9","component":"InstanceManager","file":"sled-agent/src/instance.rs:1416"} {"msg":"wait for service Some(\"oxz_propolis-server_8c58d64a-5686-44c4-ace6-adbb0c8c7347\") failed: Property not found. retry in 47.569438ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:21:04.931244167Z","hostname":"buskin","pid":1671,"instance_id":"8c1510f5-7019-4371-ba38-03f670b781e9","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Propolis SMF service is online","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:21:04.989538785Z","hostname":"buskin","pid":1671,"instance_id":"8c1510f5-7019-4371-ba38-03f670b781e9","component":"InstanceManager","file":"sled-agent/src/instance.rs:1425"} {"msg":"Propolis HTTP server online","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:21:05.033684613Z","hostname":"buskin","pid":1671,"instance_id":"8c1510f5-7019-4371-ba38-03f670b781e9","component":"InstanceManager","file":"sled-agent/src/instance.rs:1439"} {"msg":"Sending ensure request to propolis: InstanceEnsureRequest { cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAAChnDpaWFpYAACcOlpYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAoZw6WlhaWAAAnDpaWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiI4YzE1MTBmNS03MDE5LTQzNzEtYmEzOC0wM2Y2NzBiNzgxZTkiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJUFBKNEVtMU9HQ2dldUp1U1dZS1pNU3NybVo0OW5hdlVya09DUFRrc0lROCJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\"), disks: [DiskRequest { device: \"nvme\", name: \"disk-ecef3bc7e8\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 1db0dc18-2ff4-49ff-854b-668e7d436571, read_only_parent: Some(Volume { block_size: 512, id: 4a8ab656-54f6-43d3-9e93-f8956670967c, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 9438d2b2-b6ea-40a1-b98e-b23df28731d3, key: Some(\"c0Mfx7PCuxa0pnEwqQ71E/9uWhHQUVoQtA66zhe38eM=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::1a]:19002\", \"[fd00:1122:3344:101::16]:19001\", \"[fd00:1122:3344:101::14]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 1db0dc18-2ff4-49ff-854b-668e7d436571, key: Some(\"YRoHGYeAoySO1pD8lI2b9DpNtmNUw50qHwIVO2CT2Xo=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::18]:19000\", \"[fd00:1122:3344:101::14]:19002\", \"[fd00:1122:3344:101::1a]:19003\"] } }] } }], migrate: None, nics: [NetworkInterfaceRequest { name: \"vopte6\", slot: Slot(0) }], properties: InstanceProperties { bootrom_id: 00000000-0000-0000-0000-000000000000, description: \"Test description\", id: 8c1510f5-7019-4371-ba38-03f670b781e9, image_id: 00000000-0000-0000-0000-000000000000, memory: 1024, name: \"localshark\", vcpus: 2 } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:21:05.037688363Z","hostname":"buskin","pid":1671,"instance_id":"8c1510f5-7019-4371-ba38-03f670b781e9","component":"InstanceManager","file":"sled-agent/src/instance.rs:681"} {"msg":"result of instance_ensure call is Ok(InstanceEnsureResponse { migrate: None })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:21:05.18614316Z","hostname":"buskin","pid":1671,"instance_id":"8c1510f5-7019-4371-ba38-03f670b781e9","component":"InstanceManager","file":"sled-agent/src/instance.rs:683"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:21:05.186464206Z","hostname":"buskin","pid":1671,"uri":"/instances/8c1510f5-7019-4371-ba38-03f670b781e9/state","method":"PUT","req_id":"51217d6f-31cb-48b4-8b4d-c1c481481a44","remote_addr":"[fd00:1122:3344:101::a]:44490","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":7422492,"response_code":"200"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Starting), migration_status: NoMigration, time: 2024-02-26T07:21:05.186980215Z }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:21:05.187169568Z","hostname":"buskin","pid":1671,"instance_id":"8c1510f5-7019-4371-ba38-03f670b781e9","component":"InstanceManager","file":"sled-agent/src/instance.rs:565"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:21:05.187194649Z","hostname":"buskin","pid":1671,"instance_id":"8c1510f5-7019-4371-ba38-03f670b781e9","component":"InstanceManager","file":"sled-agent/src/instance.rs:587","new_vmm_state":"VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-02-26T07:21:05.186980215Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(8c58d64a-5686-44c4-ace6-adbb0c8c7347), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-02-26T07:20:57.366781248Z }","propolis_id":"8c58d64a-5686-44c4-ace6-adbb0c8c7347"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:21:05.187488014Z","hostname":"buskin","pid":1671,"instance_id":"8c1510f5-7019-4371-ba38-03f670b781e9","component":"InstanceManager","file":"sled-agent/src/instance.rs:487","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(8c58d64a-5686-44c4-ace6-adbb0c8c7347), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-02-26T07:20:57.366781248Z }, propolis_id: 8c58d64a-5686-44c4-ace6-adbb0c8c7347, vmm_state: VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-02-26T07:21:05.186980215Z } }","instance_id":"8c1510f5-7019-4371-ba38-03f670b781e9"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Running), migration_status: NoMigration, time: 2024-02-26T07:21:05.224972044Z }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:21:05.224991235Z","hostname":"buskin","pid":1671,"instance_id":"8c1510f5-7019-4371-ba38-03f670b781e9","component":"InstanceManager","file":"sled-agent/src/instance.rs:565"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:21:05.225008605Z","hostname":"buskin","pid":1671,"instance_id":"8c1510f5-7019-4371-ba38-03f670b781e9","component":"InstanceManager","file":"sled-agent/src/instance.rs:587","new_vmm_state":"VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-02-26T07:21:05.224972044Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(8c58d64a-5686-44c4-ace6-adbb0c8c7347), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-02-26T07:20:57.366781248Z }","propolis_id":"8c58d64a-5686-44c4-ace6-adbb0c8c7347"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:21:05.225017905Z","hostname":"buskin","pid":1671,"instance_id":"8c1510f5-7019-4371-ba38-03f670b781e9","component":"InstanceManager","file":"sled-agent/src/instance.rs:487","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(8c58d64a-5686-44c4-ace6-adbb0c8c7347), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-02-26T07:20:57.366781248Z }, propolis_id: 8c58d64a-5686-44c4-ace6-adbb0c8c7347, vmm_state: VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-02-26T07:21:05.224972044Z } }","instance_id":"8c1510f5-7019-4371-ba38-03f670b781e9"} {"msg":"running automatic periodic zone bundle cleanup","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:21:19.760732919Z","hostname":"buskin","pid":1671,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:299"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:21:19.7607637Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:21:19.802825571Z","hostname":"buskin","pid":1671,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:288","time_to_next_cleanup":"599.9999972s","next_cleanup":"Instant { tv_sec: 91252, tv_nsec: 418516947 }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:21:21.68524559Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:43288"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:21:21.685352202Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"f5b5af87-e403-4b64-8a9a-1263eff79bb7","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:43288","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":52,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:21:22.505206573Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:38150"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:21:22.505955806Z","hostname":"buskin","pid":1671,"uri":"/instances/8c1510f5-7019-4371-ba38-03f670b781e9/state","method":"PUT","req_id":"84040025-8335-4c2c-977e-5d2107d873ed","remote_addr":"[fd00:1122:3344:101::a]:38150","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":696,"response_code":"200"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Stopping), migration_status: NoMigration, time: 2024-02-26T07:21:22.506055668Z }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:21:22.506086098Z","hostname":"buskin","pid":1671,"instance_id":"8c1510f5-7019-4371-ba38-03f670b781e9","component":"InstanceManager","file":"sled-agent/src/instance.rs:565"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:21:22.506104698Z","hostname":"buskin","pid":1671,"instance_id":"8c1510f5-7019-4371-ba38-03f670b781e9","component":"InstanceManager","file":"sled-agent/src/instance.rs:587","new_vmm_state":"VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-02-26T07:21:22.506055668Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(8c58d64a-5686-44c4-ace6-adbb0c8c7347), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-02-26T07:20:57.366781248Z }","propolis_id":"8c58d64a-5686-44c4-ace6-adbb0c8c7347"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:21:22.506129439Z","hostname":"buskin","pid":1671,"instance_id":"8c1510f5-7019-4371-ba38-03f670b781e9","component":"InstanceManager","file":"sled-agent/src/instance.rs:487","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(8c58d64a-5686-44c4-ace6-adbb0c8c7347), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-02-26T07:20:57.366781248Z }, propolis_id: 8c58d64a-5686-44c4-ace6-adbb0c8c7347, vmm_state: VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-02-26T07:21:22.506055668Z } }","instance_id":"8c1510f5-7019-4371-ba38-03f670b781e9"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Stopped), migration_status: NoMigration, time: 2024-02-26T07:21:22.526473427Z }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:21:22.526495768Z","hostname":"buskin","pid":1671,"instance_id":"8c1510f5-7019-4371-ba38-03f670b781e9","component":"InstanceManager","file":"sled-agent/src/instance.rs:565"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:21:22.526512758Z","hostname":"buskin","pid":1671,"instance_id":"8c1510f5-7019-4371-ba38-03f670b781e9","component":"InstanceManager","file":"sled-agent/src/instance.rs:587","new_vmm_state":"VmmRuntimeState { state: Stopping, gen: Generation(6), time_updated: 2024-02-26T07:21:22.526473427Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(8c58d64a-5686-44c4-ace6-adbb0c8c7347), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-02-26T07:20:57.366781248Z }","propolis_id":"8c58d64a-5686-44c4-ace6-adbb0c8c7347"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:21:22.526536969Z","hostname":"buskin","pid":1671,"instance_id":"8c1510f5-7019-4371-ba38-03f670b781e9","component":"InstanceManager","file":"sled-agent/src/instance.rs:487","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(8c58d64a-5686-44c4-ace6-adbb0c8c7347), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-02-26T07:20:57.366781248Z }, propolis_id: 8c58d64a-5686-44c4-ace6-adbb0c8c7347, vmm_state: VmmRuntimeState { state: Stopping, gen: Generation(6), time_updated: 2024-02-26T07:21:22.526473427Z } }","instance_id":"8c1510f5-7019-4371-ba38-03f670b781e9"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Destroyed), migration_status: NoMigration, time: 2024-02-26T07:21:22.694387107Z }","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:21:22.694431348Z","hostname":"buskin","pid":1671,"instance_id":"8c1510f5-7019-4371-ba38-03f670b781e9","component":"InstanceManager","file":"sled-agent/src/instance.rs:565"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:21:22.694460178Z","hostname":"buskin","pid":1671,"instance_id":"8c1510f5-7019-4371-ba38-03f670b781e9","component":"InstanceManager","file":"sled-agent/src/instance.rs:587","new_vmm_state":"VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-02-26T07:21:22.694387107Z }","new_instance_state":"InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-02-26T07:21:22.694387107Z }","propolis_id":"8c58d64a-5686-44c4-ace6-adbb0c8c7347"} {"msg":"terminating VMM that has exited","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:21:22.694480829Z","hostname":"buskin","pid":1671,"instance_id":"8c1510f5-7019-4371-ba38-03f670b781e9","component":"InstanceManager","file":"sled-agent/src/instance.rs:602","instance_id":"8c1510f5-7019-4371-ba38-03f670b781e9"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:21:22.694487798Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:21:22.694651381Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"creating zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:21:22.694718223Z","hostname":"buskin","pid":1671,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:436","context":"ZoneBundleContext { storage_dirs: [\"/pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/debug/bundle/zone\", \"/pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/debug/bundle/zone\"], cause: TerminatedInstance, extra_log_dirs: [\"/pool/ext/14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/debug\", \"/pool/ext/24b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/debug\", \"/pool/ext/31bd71cd-4736-4a12-a387-9b74b050396f/crypt/debug\", \"/pool/ext/616b26df-e62a-4c68-b506-f4a923d8aaf7/crypt/debug\", \"/pool/ext/cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crypt/debug\", \"/pool/ext/ceb4461c-cf56-4719-ad3c-14430bfdfb60/crypt/debug\", \"/pool/ext/d462a7f7-b628-40fe-80ff-4e4189e2d62b/crypt/debug\", \"/pool/ext/e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/debug\", \"/pool/ext/f4b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/debug\"] }","zone_name":"oxz_propolis-server_8c58d64a-5686-44c4-ace6-adbb0c8c7347"} {"msg":"finished zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:21:23.287586092Z","hostname":"buskin","pid":1671,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:1235","metadata":"ZoneBundleMetadata { id: ZoneBundleId { zone_name: \"oxz_propolis-server_8c58d64a-5686-44c4-ace6-adbb0c8c7347\", bundle_id: 3fd044c0-8988-489a-95fc-c250fbc499eb }, time_created: 2024-02-26T07:21:22.694977147Z, version: 0, cause: TerminatedInstance }"} {"msg":"Halting and removing zone: oxz_propolis-server_8c58d64a-5686-44c4-ace6-adbb0c8c7347","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:21:23.287694094Z","hostname":"buskin","pid":1671,"instance_id":"8c1510f5-7019-4371-ba38-03f670b781e9","component":"InstanceManager","file":"sled-agent/src/instance.rs:776"} {"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:21:24.850572641Z","hostname":"buskin","pid":1671,"instance_id":"8c1510f5-7019-4371-ba38-03f670b781e9","component":"InstanceManager","file":"illumos-utils/src/zone.rs:269"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:21:24.874618835Z","hostname":"buskin","pid":1671,"instance_id":"8c1510f5-7019-4371-ba38-03f670b781e9","component":"InstanceManager","file":"sled-agent/src/instance.rs:487","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-02-26T07:21:22.694387107Z }, propolis_id: 8c58d64a-5686-44c4-ace6-adbb0c8c7347, vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(8), time_updated: 2024-02-26T07:21:24.874548584Z } }","instance_id":"8c1510f5-7019-4371-ba38-03f670b781e9"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:21:24.947061632Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:41230"} {"msg":"unset_virtual_nic_host unimplmented","v":0,"name":"SledAgent","level":40,"time":"2024-02-26T07:21:24.949588877Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:620"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:21:24.949607237Z","hostname":"buskin","pid":1671,"uri":"/v2p/032caeae-c9ce-4568-b322-19d7207fd125","method":"DELETE","req_id":"5b6e433d-e9eb-48ee-b405-53a8a2e66f5e","remote_addr":"[fd00:1122:3344:101::b]:41230","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":633,"response_code":"204"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:21:25.080225039Z","hostname":"buskin","pid":1671,"instance_id":"8c1510f5-7019-4371-ba38-03f670b781e9","component":"InstanceManager","file":"sled-agent/src/instance.rs:487","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-02-26T07:21:22.694387107Z }, propolis_id: 8c58d64a-5686-44c4-ace6-adbb0c8c7347, vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(8), time_updated: 2024-02-26T07:21:24.874548584Z } }","instance_id":"8c1510f5-7019-4371-ba38-03f670b781e9"} {"msg":"State monitoring task complete","v":0,"name":"SledAgent","level":30,"time":"2024-02-26T07:21:25.083302043Z","hostname":"buskin","pid":1671,"instance_id":"8c1510f5-7019-4371-ba38-03f670b781e9","component":"InstanceManager","file":"sled-agent/src/instance.rs:719"}