[ Mar 1 23:15:05 Enabled. ] [ Mar 1 23:15:05 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ] [ Mar 1 23:15:05 Method "start" exited with status 0. ] [ Mar 1 23:15:05 Rereading configuration. ] [ Mar 1 23:15:05 No 'refresh' method defined. Treating as :true. ] note: configured to log to "/dev/stdout" {"msg":"finding links [\"net0\", \"net1\"]","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:05.141061549Z","hostname":"buskin","pid":1671,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:247"} {"msg":"Starting mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:05.193536754Z","hostname":"buskin","pid":1671,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:254","addr-objs":"[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]"} {"msg":"Importing mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:05.193571695Z","hostname":"buskin","pid":1671,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:42","path":"/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"} {"msg":"Setting mg-ddm interfaces","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:05.303942379Z","hostname":"buskin","pid":1671,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:50","interfaces":"(\"net0/ll\" \"net1/ll\")"} {"msg":"Enabling mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:05.313231526Z","hostname":"buskin","pid":1671,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:56"} {"msg":"Ensuring zfs key directory exists","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:05.321445887Z","hostname":"buskin","pid":1671,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:264","path":"/var/run/oxide/"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:05.924008708Z","hostname":"buskin","pid":1671,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:91","prefix":"Ipv6Prefix { addr: fdb0:18c0:4d0d:9fb2::, len: 64 }"} {"msg":"Starting KeyManager","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.065385632Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:117"} {"msg":"Starting StorageManager","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.065421842Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:129"} {"msg":"Starting StorageMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.065499274Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:141"} {"msg":"Starting HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.065585297Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:162","sled_mode":"Scrimlet { asic: SoftNpuZone }"} {"msg":"DumpSetup poll loop started.","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.065599927Z","hostname":"buskin","pid":1671,"component":"DumpSetup-archival","file":"sled-agent/src/dump_setup.rs:268"} {"msg":"Creating HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.065615407Z","hostname":"buskin","pid":1671,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:609"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.065632347Z","hostname":"buskin","pid":1671,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:776"} {"msg":"Generated i86pc baseboard Pc { identifier: \"buskin\", model: \"i86pc\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.068487248Z","hostname":"buskin","pid":1671,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:535"} {"msg":"Device is not a Gimlet (i86pc), proceeding with null hardware view","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:06.068503588Z","hostname":"buskin","pid":1671,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:658"} {"msg":"Starting HardwareMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.068510478Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:176"} {"msg":"Monitoring for hardware updates","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.068515128Z","hostname":"buskin","pid":1671,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:716"} {"msg":"Upserting synthetic zpool to Storage Manager: oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.068550199Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.068562479Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.06857414Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.06857859Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.06858484Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.06858997Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.06859743Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.06860381Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.0686074Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxp_31bd71cd-4736-4a12-a387-9b74b050396f","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.068614601Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.068621951Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Waiting for boot disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.068625741Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:88"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.068633011Z","hostname":"buskin","pid":1671,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:241","underlay_network_info":"None"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: a462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal } }))","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.068703173Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Channel capacity decreased","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.256125904Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:326","current":"245","previous":"256"} {"msg":"Found boot disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.256176745Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:90"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: b462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal } }))","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.256183846Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } }))","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.427786989Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }))","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.427835091Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }))","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.427846691Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }))","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.427851511Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }))","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.427866211Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } }))","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.427876252Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } }))","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.427887482Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }))","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.427894132Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } }))","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.427912112Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received DisksChanged({})","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.427918033Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Channel capacity increased","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.427947023Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:336","current":"255","previous":"245"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.427968664Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.428075596Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:111","resources":"StorageResources { disks: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: a462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal } }), Pool { name: ZpoolName { id: a462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal }, info: ZpoolInfo { name: \"oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b\", size: 20937965568, allocated: 499200, free: 20937466368, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: b462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal } }), Pool { name: ZpoolName { id: b462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal }, info: ZpoolInfo { name: \"oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b\", size: 20937965568, allocated: 497664, free: 20937467904, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" } })} }"} {"msg":"Starting Bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.428108047Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:200"} {"msg":"No ledger in /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.428197479Z","hostname":"buskin","pid":1671,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:187"} {"msg":"No ledger in /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.4282353Z","hostname":"buskin","pid":1671,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:187"} {"msg":"No ledger found. Loading Fsm::State as Uninitialized","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.42824147Z","hostname":"buskin","pid":1671,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:83"} {"msg":"No ledger in /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.428262691Z","hostname":"buskin","pid":1671,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:187"} {"msg":"No ledger in /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.428276651Z","hostname":"buskin","pid":1671,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:187"} {"msg":"No ledger found for network config","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.428281671Z","hostname":"buskin","pid":1671,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:146"} {"msg":"Starting Bootstore DDMD poller","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.428319792Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:205"} {"msg":"Starting ZoneBundler related tasks","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.428341233Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:220"} {"msg":"Creating ServiceManager","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.442805517Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:700"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.442834758Z","hostname":"buskin","pid":1671,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:288","time_to_next_cleanup":"599.999935538s","next_cleanup":"Instant { tv_sec: 1625, tv_nsec: 276681643 }"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.442883699Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3407"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.44290562Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"No ledger in /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.442975451Z","hostname":"buskin","pid":1671,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:187"} {"msg":"No ledger in /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.444886208Z","hostname":"buskin","pid":1671,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:187"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.445865442Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:195"} {"msg":"Started listening","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.445953544Z","hostname":"buskin","pid":1671,"component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:44","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Injecting RSS configuration: RackInitializeRequest {\n trust_quorum_peers: None,\n bootstrap_discovery: OnlyOurs,\n ntp_servers: [\n \"0.pool.ntp.org\",\n ],\n dns_servers: [\n 1.1.1.1,\n 9.9.9.9,\n ],\n internal_services_ip_pool_ranges: [\n V4(\n Ipv4Range {\n first: 10.151.1.164,\n last: 10.151.1.173,\n },\n ),\n ],\n external_dns_ips: [\n 10.151.1.164,\n 10.151.1.165,\n ],\n external_dns_zone_name: \"oxide.test\",\n external_certificates: \"\",\n recovery_silo: RecoverySiloConfig {\n silo_name: Name(\n \"recovery\",\n ),\n user_name: UserId(\n \"recovery\",\n ),\n user_password_hash: NewPasswordHash(\n PasswordHashString {\n string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n encoding: B64,\n },\n ),\n },\n rack_network_config: RackNetworkConfigV1 {\n rack_subnet: Ipv6Network {\n addr: fd00:1122:3344:100::,\n prefix: 56,\n },\n infra_ip_first: 192.168.1.30,\n infra_ip_last: 192.168.1.30,\n ports: [\n PortConfigV1 {\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Network {\n addr: 0.0.0.0,\n prefix: 0,\n },\n ),\n nexthop: 10.151.1.100,\n },\n ],\n addresses: [\n V4(\n Ipv4Network {\n addr: 10.151.1.174,\n prefix: 24,\n },\n ),\n ],\n switch: Switch0,\n port: \"qsfp0\",\n uplink_port_speed: Speed40G,\n uplink_port_fec: None,\n bgp_peers: [],\n autoneg: false,\n },\n ],\n bgp: [],\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.446229541Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:800"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.446236311Z","hostname":"buskin","pid":1671,"component":"DnsResolver","component":"RSS","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.446446496Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"No ledger in /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.446713923Z","hostname":"buskin","pid":1671,"component":"RSS","file":"common/src/ledger.rs:187"} {"msg":"No ledger in /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.446739934Z","hostname":"buskin","pid":1671,"component":"RSS","file":"common/src/ledger.rs:187"} {"msg":"RSS configuration has not been fully applied yet","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.446747634Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:860"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.446771524Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"No ledger in /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.446786535Z","hostname":"buskin","pid":1671,"component":"RSS","file":"common/src/ledger.rs:187"} {"msg":"No ledger in /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.446796005Z","hostname":"buskin","pid":1671,"component":"RSS","file":"common/src/ledger.rs:187"} {"msg":"Creating new allocation plan","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.446809375Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:901"} {"msg":"Serializing plan","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.446812865Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:118"} {"msg":"Creating plan for the sled at fdb0:18c0:4d0d:9fb2::1","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.446816355Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:91"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.446893797Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.446908498Z","hostname":"buskin","pid":1671,"component":"RSS","file":"common/src/ledger.rs:198"} {"msg":"Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.447230515Z","hostname":"buskin","pid":1671,"component":"RSS","file":"common/src/ledger.rs:198"} {"msg":"Sled plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.447379539Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:138"} {"msg":"Writing Rack Network Configuration to bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.447388509Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:934"} {"msg":"Attempting to update network config with generation: 1, current_generation: 0","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.44742581Z","hostname":"buskin","pid":1671,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.4474336Z","hostname":"buskin","pid":1671,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:198"} {"msg":"Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.447588444Z","hostname":"buskin","pid":1671,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:198"} {"msg":"Broadcasting network config with generation 1","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.447724458Z","hostname":"buskin","pid":1671,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"Received initialization request from RSS","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.447766599Z","hostname":"buskin","pid":1671,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:217","target_sled":"[fdb0:18c0:4d0d:9fb2::1]:12346","request":"StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: cc74b0a2-490e-4424-adad-bb6a277ade47, rack_id: 5d4f5e32-9bf1-4871-a9fd-8e138cda5d0f, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net(Ipv6Network { addr: fd00:1122:3344:101::, prefix: 64 }) } } }"} {"msg":"Accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.447893272Z","hostname":"buskin","pid":1671,"remote_addr":"[fdb0:18c0:4d0d:9fb2::1]:45375","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:66"} {"msg":"Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: cc74b0a2-490e-4424-adad-bb6a277ade47, rack_id: 5d4f5e32-9bf1-4871-a9fd-8e138cda5d0f, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net(Ipv6Network { addr: fd00:1122:3344:101::, prefix: 64 }) } } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.448010625Z","hostname":"buskin","pid":1671,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:364"} {"msg":"KeyManager: using hardcoded secret retriever","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.448019715Z","hostname":"buskin","pid":1671,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:379"} {"msg":"setting up sled agent server","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.448033885Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/server.rs:47"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.448039005Z","hostname":"buskin","pid":1671,"component":"DnsResolver","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"Received KeyManagerReady","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.448085987Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Attempting to add queued disks","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.448092617Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:394","num_disks":"9"} {"msg":"Enabling switch zone (new)","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.450463395Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3635"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.456495282Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:327","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\",\n model: \"synthetic-model\",\n}"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.456546154Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:329","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.456552514Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:333","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.456574735Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:335","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.456685817Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:37"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.456691317Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:347"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.486398705Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:56"} {"msg":"Automatically destroying dataset: oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.494159825Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:389"} {"msg":"SledAgent::new(..) starting","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.527609335Z","hostname":"buskin","pid":1671,"sled_id":"cc74b0a2-490e-4424-adad-bb6a277ade47","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:342"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.527648216Z","hostname":"buskin","pid":1671,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:91","prefix":"Ipv6Prefix { addr: fd00:1122:3344:101::, len: 64 }"} {"msg":"Looking for unencrypted datasets in oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.599472485Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:474"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.621872894Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:327","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.621940985Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:329","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.621953946Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:333","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.621995977Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:335","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.62213643Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:37"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.62214706Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:347"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.649293276Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:56"} {"msg":"Automatically destroying dataset: oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.65724082Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:389"} {"msg":"Looking for unencrypted datasets in oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.761451163Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:474"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.784152949Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:327","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.78420164Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:329","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.784213721Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:333","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.784253471Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:335","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.784378465Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:37"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.784388275Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:347"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.812062433Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:56"} {"msg":"Automatically destroying dataset: oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.819738651Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:389"} {"msg":"Looking for unencrypted datasets in oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.92747894Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:474"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.950260848Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:327","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.950295449Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:329","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.9503196Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:333","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.95034663Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:335","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.950489214Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:37"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.950498584Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:347"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.979271858Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:56"} {"msg":"Automatically destroying dataset: oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:06.987247443Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:389"} {"msg":"Looking for unencrypted datasets in oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:07.095217868Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:474"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:07.118597461Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:327","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\",\n model: \"synthetic-model\",\n}"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:07.118634502Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:329","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:07.118641632Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:333","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:07.118660803Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:335","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:07.118754615Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:37"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:07.118762595Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:347"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:07.145990262Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:56"} {"msg":"Automatically destroying dataset: oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:07.154409998Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:389"} {"msg":"Looking for unencrypted datasets in oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:07.260517808Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:474"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:07.283779808Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:327","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\",\n model: \"synthetic-model\",\n}"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:07.283824159Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:329","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:07.283831469Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:333","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:07.28385056Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:335","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:07.283960582Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:37"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:07.283967342Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:347"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:07.311262511Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:56"} {"msg":"Automatically destroying dataset: oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:07.318753095Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:389"} {"msg":"Looking for unencrypted datasets in oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:07.4218551Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:474"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:07.443826048Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:327","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:07.443863279Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:329","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:07.4439088Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:333","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:07.443993743Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:335","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:07.444099305Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:37"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:07.444105605Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:347"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:07.471151968Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:56"} {"msg":"Automatically destroying dataset: oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:07.478553819Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:389"} {"msg":"Looking for unencrypted datasets in oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:07.580883896Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:474"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:07.603590122Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:327","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\",\n model: \"synthetic-model\",\n}"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:07.603650403Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:329","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:07.603659464Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:333","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:07.603670144Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:335","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:07.603758616Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:37"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:07.603766506Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:347"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:07.632254484Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:56"} {"msg":"Automatically destroying dataset: oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:07.639686886Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:389"} {"msg":"Looking for unencrypted datasets in oxp_31bd71cd-4736-4a12-a387-9b74b050396f","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:07.744030622Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:474"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:07.765914589Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:327","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\",\n model: \"synthetic-model\",\n}"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:07.76596486Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:329","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:07.76597186Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:333","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:07.766018161Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:335","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:07.766158784Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:37"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:07.766165275Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:347"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:07.793883904Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:56"} {"msg":"Automatically destroying dataset: oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:07.801198603Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:389"} {"msg":"Looking for unencrypted datasets in oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:07.904063473Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:474"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:07.919324367Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:07.919371578Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Requested swap device of size 64 GiB","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:07.919379078Z","hostname":"buskin","pid":1671,"sled_id":"cc74b0a2-490e-4424-adad-bb6a277ade47","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:354"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:07.922084944Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:111","resources":"StorageResources { disks: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }), Pool { name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, info: ZpoolInfo { name: \"oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03\", size: 20937965568, allocated: 772608, free: 20937192960, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }), Pool { name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, info: ZpoolInfo { name: \"oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03\", size: 20937965568, allocated: 778752, free: 20937186816, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }), Pool { name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, info: ZpoolInfo { name: \"oxp_31bd71cd-4736-4a12-a387-9b74b050396f\", size: 20937965568, allocated: 820224, free: 20937145344, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } }), Pool { name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, info: ZpoolInfo { name: \"oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7\", size: 20937965568, allocated: 771072, free: 20937194496, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: a462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal } }), Pool { name: ZpoolName { id: a462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal }, info: ZpoolInfo { name: \"oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b\", size: 20937965568, allocated: 499200, free: 20937466368, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: b462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal } }), Pool { name: ZpoolName { id: b462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal }, info: ZpoolInfo { name: \"oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b\", size: 20937965568, allocated: 497664, free: 20937467904, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } }), Pool { name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, info: ZpoolInfo { name: \"oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", size: 20937965568, allocated: 771072, free: 20937194496, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } }), Pool { name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, info: ZpoolInfo { name: \"oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60\", size: 20937965568, allocated: 774144, free: 20937191424, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } }), Pool { name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, info: ZpoolInfo { name: \"oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b\", size: 20937965568, allocated: 772608, free: 20937192960, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }), Pool { name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, info: ZpoolInfo { name: \"oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", size: 20937965568, allocated: 1728000, free: 20936237568, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }), Pool { name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, info: ZpoolInfo { name: \"oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", size: 20937965568, allocated: 820224, free: 20937145344, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" } })} }"} {"msg":"attempting to create encrypted zvol: name=\"oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:07.941961581Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/swap_device.rs:153"} {"msg":"Configuring new Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:07.944663087Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:07.961194832Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"successfully created encrypted zvol: name=\"oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:07.964304018Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/swap_device.rs:249"} {"msg":"adding swap device: swapname=\"/dev/zvol/dsk/oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b/swap\"","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:07.96434737Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/swap_device.rs:96"} {"msg":"Mounting backing filesystems","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:07.969948397Z","hostname":"buskin","pid":1671,"sled_id":"cc74b0a2-490e-4424-adad-bb6a277ade47","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:369"} {"msg":"Processing fmd","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:07.969960037Z","hostname":"buskin","pid":1671,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:126"} {"msg":"Ensuring dataset oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b/backing/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:07.969965247Z","hostname":"buskin","pid":1671,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:136"} {"msg":"Stopping service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:08.042064903Z","hostname":"buskin","pid":1671,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:170"} {"msg":"Mounting oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b/backing/fmd on /var/fm/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:08.260877954Z","hostname":"buskin","pid":1671,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:178"} {"msg":"Ensuring directory /var/fm/fmd/rsrc","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:08.273031381Z","hostname":"buskin","pid":1671,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Ensuring directory /var/fm/fmd/ckpt","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:08.273175475Z","hostname":"buskin","pid":1671,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Ensuring directory /var/fm/fmd/xprt","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:08.273216896Z","hostname":"buskin","pid":1671,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Starting service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:08.273259867Z","hostname":"buskin","pid":1671,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:192"} {"msg":"Ensuring contract reaper thread","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:08.439384936Z","hostname":"buskin","pid":1671,"file":"illumos-utils/src/running_zone.rs:163"} {"msg":"using '[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]' as data links for xde driver","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:08.977882967Z","hostname":"buskin","pid":1671,"sled_id":"cc74b0a2-490e-4424-adad-bb6a277ade47","component":"SledAgent","file":"illumos-utils/src/opte/illumos.rs:90"} {"msg":"Underlay Available","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:08.97842374Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:122","sled_id":"cc74b0a2-490e-4424-adad-bb6a277ade47"} {"msg":"50% of 102952026112 physical ram = 51476013056 bytes)","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:08.978442321Z","hostname":"buskin","pid":1671,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:206"} {"msg":"Setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:08.978449061Z","hostname":"buskin","pid":1671,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:214"} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: cc74b0a2-490e-4424-adad-bb6a277ade47, variant: M2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:08.994376721Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", sled_id: cc74b0a2-490e-4424-adad-bb6a277ade47, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:08.994594186Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: cc74b0a2-490e-4424-adad-bb6a277ade47, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:08.994918904Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:08.99516696Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cc74b0a2-490e-4424-adad-bb6a277ade47/zpools/24b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:08.995479408Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cc74b0a2-490e-4424-adad-bb6a277ade47/zpools/b462a7f7-b628-40fe-80ff-4e4189e2d62b): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:08.995792406Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cc74b0a2-490e-4424-adad-bb6a277ade47/zpools/e4b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: cc74b0a2-490e-4424-adad-bb6a277ade47, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:08.99596224Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: cc74b0a2-490e-4424-adad-bb6a277ade47, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:08.996104543Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", sled_id: cc74b0a2-490e-4424-adad-bb6a277ade47, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:08.996235427Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", sled_id: cc74b0a2-490e-4424-adad-bb6a277ade47, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:08.99636089Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: cc74b0a2-490e-4424-adad-bb6a277ade47, variant: M2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:08.996489333Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", sled_id: cc74b0a2-490e-4424-adad-bb6a277ade47, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:08.996628436Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: cc74b0a2-490e-4424-adad-bb6a277ade47, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:08.996758879Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: cc74b0a2-490e-4424-adad-bb6a277ade47, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:08.996898663Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:08.997032316Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cc74b0a2-490e-4424-adad-bb6a277ade47/zpools/14b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxp_31bd71cd-4736-4a12-a387-9b74b050396f on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:08.997161309Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cc74b0a2-490e-4424-adad-bb6a277ade47/zpools/31bd71cd-4736-4a12-a387-9b74b050396f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:08.997281742Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cc74b0a2-490e-4424-adad-bb6a277ade47/zpools/616b26df-e62a-4c68-b506-f4a923d8aaf7): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:08.997428876Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cc74b0a2-490e-4424-adad-bb6a277ade47/zpools/a462a7f7-b628-40fe-80ff-4e4189e2d62b): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:08.99758014Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cc74b0a2-490e-4424-adad-bb6a277ade47/zpools/cd70d7f6-2354-4bf2-8012-55bf9eaf7930): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:08.997726323Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cc74b0a2-490e-4424-adad-bb6a277ade47/zpools/ceb4461c-cf56-4719-ad3c-14430bfdfb60): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:08.997869647Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cc74b0a2-490e-4424-adad-bb6a277ade47/zpools/d462a7f7-b628-40fe-80ff-4e4189e2d62b): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:08.997989379Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cc74b0a2-490e-4424-adad-bb6a277ade47/zpools/f4b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:11.634386741Z","hostname":"buskin","pid":1671,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 36.94182ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:12.039835873Z","hostname":"buskin","pid":1671,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 111.082935ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:12.235026444Z","hostname":"buskin","pid":1671,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 101.079343ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:12.432747898Z","hostname":"buskin","pid":1671,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 566.820962ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:12.624192238Z","hostname":"buskin","pid":1671,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 579.991254ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:13.245269391Z","hostname":"buskin","pid":1671,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.434468148s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:13.880469421Z","hostname":"buskin","pid":1671,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 964.502875ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:15.366598176Z","hostname":"buskin","pid":1671,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 855.547566ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:16.476636517Z","hostname":"buskin","pid":1671,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring tfportrear0_0/ll exists in zone","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:18.264859031Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1949"} {"msg":"sled agent started","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:19.370228349Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:981","underlay_address":"fd00:1122:3344:101::1"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:19.37029667Z","hostname":"buskin","pid":1671,"component":"DnsResolver","component":"ServiceManager","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:19.429194123Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:195"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:19.429234584Z","hostname":"buskin","pid":1671,"sled_id":"cc74b0a2-490e-4424-adad-bb6a277ade47","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"cc74b0a2-490e-4424-adad-bb6a277ade47"} {"msg":"Sled Agent loaded; recording configuration","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:19.429245384Z","hostname":"buskin","pid":1671,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:424"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:19.429262384Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:19.429294725Z","hostname":"buskin","pid":1671,"component":"SledAgentMain","file":"common/src/ledger.rs:198"} {"msg":"Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:19.429596643Z","hostname":"buskin","pid":1671,"component":"SledAgentMain","file":"common/src/ledger.rs:198"} {"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:19.429700385Z","hostname":"buskin","pid":1671,"sled_id":"cc74b0a2-490e-4424-adad-bb6a277ade47","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1199","retry_after":"259.831728ms","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"Sled Agent Started","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:19.429758357Z","hostname":"buskin","pid":1671,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:137"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:19.429796258Z","hostname":"buskin","pid":1671,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:241","underlay_network_info":"Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV1 { rack_subnet: Ipv6Network { addr: fd00:1122:3344:100::, prefix: 56 }, infra_ip_first: 192.168.1.30, infra_ip_last: 192.168.1.30, ports: [PortConfigV1 { routes: [RouteConfig { destination: V4(Ipv4Network { addr: 0.0.0.0, prefix: 0 }), nexthop: 10.151.1.100 }], addresses: [V4(Ipv4Network { addr: 10.151.1.174, prefix: 24 })], switch: Switch0, port: \"qsfp0\", uplink_port_speed: Speed40G, uplink_port_fec: None, bgp_peers: [], autoneg: false }], bgp: [] })))"} {"msg":"Connection closed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:19.429803388Z","hostname":"buskin","pid":1671,"remote_addr":"[fdb0:18c0:4d0d:9fb2::1]:45375","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:73"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:19.429809038Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3407"} {"msg":"Peer agent initialized","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:19.42988378Z","hostname":"buskin","pid":1671,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:108","peer":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Initialized sled agent","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:19.42989128Z","hostname":"buskin","pid":1671,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:232","target_sled":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:19.433121979Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"No ledger in /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/rss-service-plan-v2.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:19.433228122Z","hostname":"buskin","pid":1671,"component":"RSS","file":"common/src/ledger.rs:187"} {"msg":"No ledger in /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/rss-service-plan-v2.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:19.433251752Z","hostname":"buskin","pid":1671,"component":"RSS","file":"common/src/ledger.rs:187"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:19.433316404Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"failed to notify nexus about sled agent","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:19.434028061Z","hostname":"buskin","pid":1671,"sled_id":"cc74b0a2-490e-4424-adad-bb6a277ade47","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:735","error":"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cc74b0a2-490e-4424-adad-bb6a277ade47): error trying to connect: dns error: proto error: io error: No route to host (os error 148)"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:19.465690187Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:49315"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:19.465927303Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:19.466025585Z","hostname":"buskin","pid":1671,"uri":"/zpools","method":"GET","req_id":"b8537979-813d-41d3-891a-0e2666884589","remote_addr":"[fd00:1122:3344:101::1]:49315","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":190,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:19.49521976Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:39075"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:19.495335103Z","hostname":"buskin","pid":1671,"uri":"/sled-role","method":"GET","req_id":"11b2b0fc-bf38-444a-91e8-fc52422fa257","remote_addr":"[fd00:1122:3344:101::1]:39075","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":62,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:19.497514236Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.rss-service-plan-v2.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:19.497522477Z","hostname":"buskin","pid":1671,"component":"RSS","file":"common/src/ledger.rs:198"} {"msg":"Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.rss-service-plan-v2.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:19.497796354Z","hostname":"buskin","pid":1671,"component":"RSS","file":"common/src/ledger.rs:198"} {"msg":"Service plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:19.497986848Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/plan/service.rs:741"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:19.529011248Z","hostname":"buskin","pid":1671,"sled_id":"cc74b0a2-490e-4424-adad-bb6a277ade47","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"cc74b0a2-490e-4424-adad-bb6a277ade47"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: 720f4076-e581-438f-9cb1-67dafb6cb05c, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:19.529808908Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:19.530028003Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:43477"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 720f4076-e581-438f-9cb1-67dafb6cb05c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:19.530277949Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 720f4076-e581-438f-9cb1-67dafb6cb05c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:19.530289169Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:19.583487933Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:19.583527543Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:19.633239221Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:19.633280632Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:19.680224992Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"No ledger in /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:19.680266183Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:187"} {"msg":"No ledger in /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:19.680283924Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:187"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:19.680330945Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:19.695411724Z","hostname":"buskin","pid":1671,"sled_id":"cc74b0a2-490e-4424-adad-bb6a277ade47","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1199","retry_after":"427.846032ms","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:19.710358181Z","hostname":"buskin","pid":1671,"sled_id":"cc74b0a2-490e-4424-adad-bb6a277ade47","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"cc74b0a2-490e-4424-adad-bb6a277ade47"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:19.723754009Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:19.72380231Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:19.72380855Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_2698336d-5a4b-4958-8e5d-8a26f208fcce","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:19.777350342Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_720f4076-e581-438f-9cb1-67dafb6cb05c","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:19.781323889Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_0fb91602-87a5-4c5e-baa6-f0c1b28cef23","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:19.784916347Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_internal_dns_2698336d-5a4b-4958-8e5d-8a26f208fcce","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:19.796668965Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_720f4076-e581-438f-9cb1-67dafb6cb05c","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:19.80137494Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_0fb91602-87a5-4c5e-baa6-f0c1b28cef23","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:19.805635164Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:19.822644951Z","hostname":"buskin","pid":1671,"sled_id":"cc74b0a2-490e-4424-adad-bb6a277ade47","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"cc74b0a2-490e-4424-adad-bb6a277ade47"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:20.074206194Z","hostname":"buskin","pid":1671,"sled_id":"cc74b0a2-490e-4424-adad-bb6a277ade47","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"cc74b0a2-490e-4424-adad-bb6a277ade47"} {"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:20.124606858Z","hostname":"buskin","pid":1671,"sled_id":"cc74b0a2-490e-4424-adad-bb6a277ade47","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1199","retry_after":"572.98678ms","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:20.333559587Z","hostname":"buskin","pid":1671,"sled_id":"cc74b0a2-490e-4424-adad-bb6a277ade47","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"cc74b0a2-490e-4424-adad-bb6a277ade47"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:20.559671346Z","hostname":"buskin","pid":1671,"sled_id":"cc74b0a2-490e-4424-adad-bb6a277ade47","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"cc74b0a2-490e-4424-adad-bb6a277ade47"} {"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:20.699784448Z","hostname":"buskin","pid":1671,"sled_id":"cc74b0a2-490e-4424-adad-bb6a277ade47","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1199","retry_after":"2.552719352s","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:20.755688088Z","hostname":"buskin","pid":1671,"sled_id":"cc74b0a2-490e-4424-adad-bb6a277ade47","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"cc74b0a2-490e-4424-adad-bb6a277ade47"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:20.939314646Z","hostname":"buskin","pid":1671,"sled_id":"cc74b0a2-490e-4424-adad-bb6a277ade47","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"cc74b0a2-490e-4424-adad-bb6a277ade47"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:21.264618194Z","hostname":"buskin","pid":1671,"sled_id":"cc74b0a2-490e-4424-adad-bb6a277ade47","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"cc74b0a2-490e-4424-adad-bb6a277ade47"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:21.48692454Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_2698336d-5a4b-4958-8e5d-8a26f208fcce","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:21.562772008Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_720f4076-e581-438f-9cb1-67dafb6cb05c","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:21.579402085Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_0fb91602-87a5-4c5e-baa6-f0c1b28cef23","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_internal_dns_2698336d-5a4b-4958-8e5d-8a26f208fcce\") failed: Property not found. retry in 64.27033ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:21.629234966Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_2698336d-5a4b-4958-8e5d-8a26f208fcce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_720f4076-e581-438f-9cb1-67dafb6cb05c\") failed: Property not found. retry in 60.687188ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:21.681369103Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_720f4076-e581-438f-9cb1-67dafb6cb05c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_2698336d-5a4b-4958-8e5d-8a26f208fcce\") failed: Property not found. retry in 84.650303ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:21.716497144Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_2698336d-5a4b-4958-8e5d-8a26f208fcce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_0fb91602-87a5-4c5e-baa6-f0c1b28cef23\") failed: Property not found. retry in 63.392428ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:21.731319197Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_0fb91602-87a5-4c5e-baa6-f0c1b28cef23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_720f4076-e581-438f-9cb1-67dafb6cb05c\") failed: Property not found. retry in 96.21186ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:21.773051519Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_720f4076-e581-438f-9cb1-67dafb6cb05c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_0fb91602-87a5-4c5e-baa6-f0c1b28cef23\") failed: Property not found. retry in 138.451877ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:21.927082092Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_0fb91602-87a5-4c5e-baa6-f0c1b28cef23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:21.927147044Z","hostname":"buskin","pid":1671,"sled_id":"cc74b0a2-490e-4424-adad-bb6a277ade47","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"cc74b0a2-490e-4424-adad-bb6a277ade47"} {"msg":"wait for service Some(\"oxz_internal_dns_2698336d-5a4b-4958-8e5d-8a26f208fcce\") failed: Property not found. retry in 198.746984ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:21.9490411Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_2698336d-5a4b-4958-8e5d-8a26f208fcce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_720f4076-e581-438f-9cb1-67dafb6cb05c\") failed: Property not found. retry in 204.476361ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:21.961858314Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_720f4076-e581-438f-9cb1-67dafb6cb05c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_0fb91602-87a5-4c5e-baa6-f0c1b28cef23\") failed: Property not found. retry in 264.956666ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:22.119110275Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_0fb91602-87a5-4c5e-baa6-f0c1b28cef23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_2698336d-5a4b-4958-8e5d-8a26f208fcce\") failed: Property not found. retry in 243.124486ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:22.182303254Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_2698336d-5a4b-4958-8e5d-8a26f208fcce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_720f4076-e581-438f-9cb1-67dafb6cb05c\") failed: Property not found. retry in 414.541062ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:22.201207847Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_720f4076-e581-438f-9cb1-67dafb6cb05c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_0fb91602-87a5-4c5e-baa6-f0c1b28cef23\") failed: Property not found. retry in 329.024127ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:22.394619095Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_0fb91602-87a5-4c5e-baa6-f0c1b28cef23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_2698336d-5a4b-4958-8e5d-8a26f208fcce\") failed: Property not found. retry in 497.584034ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:22.434974473Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_2698336d-5a4b-4958-8e5d-8a26f208fcce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:22.447136251Z","hostname":"buskin","pid":1671,"sled_id":"cc74b0a2-490e-4424-adad-bb6a277ade47","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"cc74b0a2-490e-4424-adad-bb6a277ade47"} {"msg":"wait for service Some(\"oxz_internal_dns_720f4076-e581-438f-9cb1-67dafb6cb05c\") failed: Property not found. retry in 835.996254ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:22.625100671Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_720f4076-e581-438f-9cb1-67dafb6cb05c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_0fb91602-87a5-4c5e-baa6-f0c1b28cef23\") failed: Property not found. retry in 986.563306ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:22.733393264Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_0fb91602-87a5-4c5e-baa6-f0c1b28cef23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_2698336d-5a4b-4958-8e5d-8a26f208fcce\") failed: Property not found. retry in 709.11387ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:22.942353513Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_2698336d-5a4b-4958-8e5d-8a26f208fcce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring bootstrap address fdb0:18c0:4d0d:9fb2::2 exists in switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:23.081860869Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1966"} {"msg":"Adding bootstrap address","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:23.08189034Z","hostname":"buskin","pid":1671,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:626"} {"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:23.2545812Z","hostname":"buskin","pid":1671,"sled_id":"cc74b0a2-490e-4424-adad-bb6a277ade47","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1199","retry_after":"4.937609678s","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"wait for service Some(\"oxz_internal_dns_720f4076-e581-438f-9cb1-67dafb6cb05c\") failed: Property not found. retry in 1.177447533s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:23.470797897Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_720f4076-e581-438f-9cb1-67dafb6cb05c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:23.529110895Z","hostname":"buskin","pid":1671,"sled_id":"cc74b0a2-490e-4424-adad-bb6a277ade47","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"cc74b0a2-490e-4424-adad-bb6a277ade47"} {"msg":"wait for service Some(\"oxz_internal_dns_2698336d-5a4b-4958-8e5d-8a26f208fcce\") failed: Property not found. retry in 1.438663974s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:23.661037307Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_2698336d-5a4b-4958-8e5d-8a26f208fcce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_0fb91602-87a5-4c5e-baa6-f0c1b28cef23\") failed: Property not found. retry in 1.134345605s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:23.72932336Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_0fb91602-87a5-4c5e-baa6-f0c1b28cef23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:24.068355575Z","hostname":"buskin","pid":1671,"sled_id":"cc74b0a2-490e-4424-adad-bb6a277ade47","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"cc74b0a2-490e-4424-adad-bb6a277ade47"} {"msg":"wait for service Some(\"oxz_internal_dns_720f4076-e581-438f-9cb1-67dafb6cb05c\") failed: Property not found. retry in 1.397838135s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:24.657063386Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_720f4076-e581-438f-9cb1-67dafb6cb05c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:24.847236925Z","hostname":"buskin","pid":1671,"sled_id":"cc74b0a2-490e-4424-adad-bb6a277ade47","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"cc74b0a2-490e-4424-adad-bb6a277ade47"} {"msg":"wait for service Some(\"oxz_internal_dns_0fb91602-87a5-4c5e-baa6-f0c1b28cef23\") failed: Property not found. retry in 908.929752ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:24.87357901Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_0fb91602-87a5-4c5e-baa6-f0c1b28cef23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_2698336d-5a4b-4958-8e5d-8a26f208fcce\") failed: Property not found. retry in 838.352879ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:25.110252517Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_2698336d-5a4b-4958-8e5d-8a26f208fcce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_0fb91602-87a5-4c5e-baa6-f0c1b28cef23\") failed: Property not found. retry in 1.384403438s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:25.791013933Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_0fb91602-87a5-4c5e-baa6-f0c1b28cef23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_2698336d-5a4b-4958-8e5d-8a26f208fcce\") failed: Property not found. retry in 778.627573ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:25.956916558Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_2698336d-5a4b-4958-8e5d-8a26f208fcce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_720f4076-e581-438f-9cb1-67dafb6cb05c\") failed: Property not found. retry in 1.257280659s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:26.06319599Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_720f4076-e581-438f-9cb1-67dafb6cb05c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_2698336d-5a4b-4958-8e5d-8a26f208fcce\") failed: Property not found. retry in 579.128133ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:26.744469979Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_2698336d-5a4b-4958-8e5d-8a26f208fcce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:26.759542799Z","hostname":"buskin","pid":1671,"sled_id":"cc74b0a2-490e-4424-adad-bb6a277ade47","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"cc74b0a2-490e-4424-adad-bb6a277ade47"} {"msg":"wait for service Some(\"oxz_internal_dns_0fb91602-87a5-4c5e-baa6-f0c1b28cef23\") failed: Property not found. retry in 980.784299ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:27.185076062Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_0fb91602-87a5-4c5e-baa6-f0c1b28cef23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_720f4076-e581-438f-9cb1-67dafb6cb05c\") failed: Property not found. retry in 1.456118028s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:27.330618967Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_720f4076-e581-438f-9cb1-67dafb6cb05c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_2698336d-5a4b-4958-8e5d-8a26f208fcce\") failed: Property not found. retry in 1.05964701s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:27.337944207Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_2698336d-5a4b-4958-8e5d-8a26f208fcce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:27.563504592Z","hostname":"buskin","pid":1671,"sled_id":"cc74b0a2-490e-4424-adad-bb6a277ade47","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"cc74b0a2-490e-4424-adad-bb6a277ade47"} {"msg":"Forwarding bootstrap traffic via oxBootstrap0 to fe80::8:20ff:fe52:618f","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:27.61974032Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1973"} {"msg":"Zone not using gateway (even though underlay is up)","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:27.626463014Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2034"} {"msg":"Setting up dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:27.753614589Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2466"} {"msg":"Setting up MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:28.044187956Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2345"} {"msg":"Ensuring address fd00:1122:3344:2::1 exists","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:28.206676787Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2002"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:2::1, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:28.206733128Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_0fb91602-87a5-4c5e-baa6-f0c1b28cef23","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"Setting up wicketd service","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:28.406078882Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2380"} {"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:28.464963234Z","hostname":"buskin","pid":1671,"sled_id":"cc74b0a2-490e-4424-adad-bb6a277ade47","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1199","retry_after":"10.22320938s","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"Setting up mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:28.620091804Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2647"} {"msg":"Ensuring address fd00:1122:3344:2::1 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:28.759971351Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2010"} {"msg":"Zone not using gateway (even though underlay is up)","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:28.760012692Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2034"} {"msg":"Setting up mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:28.766401259Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2652"} {"msg":"Setting up tfport service","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:28.94935343Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2586"} {"msg":"Setting up Simulated SP service","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:29.116123386Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2374"} {"msg":"Re-enabling running switch zone (new address)","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:29.126461699Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3659","new":"[fd00:1122:3344:101::2, ::1]","old":"[::1]"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:29.126490129Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3675"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:29.1264993Z","hostname":"buskin","pid":1671,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"Setting up internal-dns service","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:29.434783832Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2187"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:29.672609998Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3683"} {"msg":"configuring dendrite zone","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:29.678312897Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3739"} {"msg":"configuring wicketd","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:29.858309417Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3779","rack_subnet":"fd00:1122:3344::"} {"msg":"Determining physical location of our switch zone at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:29.912248048Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:360"} {"msg":"Failed to get switch ID from MGS (retrying in 35.963784ms)","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:29.953358515Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:379","error":"Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1317e546-8ea8-4a89-b370-b5fc7a44a082\", \"content-length\": \"152\", \"date\": \"Fri, 01 Mar 2024 23:15:29 GMT\"}; value: Error { error_code: Some(\"DiscoveryNotYetComplete\"), message: \"discovery process not yet complete\", request_id: \"1317e546-8ea8-4a89-b370-b5fc7a44a082\" }"} {"msg":"Failed to get switch ID from MGS (retrying in 76.468579ms)","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:29.990257718Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:379","error":"Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"fcf39ca4-3de3-4c35-a7f8-aa2d4b6c73cc\", \"content-length\": \"152\", \"date\": \"Fri, 01 Mar 2024 23:15:29 GMT\"}; value: Error { error_code: Some(\"DiscoveryNotYetComplete\"), message: \"discovery process not yet complete\", request_id: \"fcf39ca4-3de3-4c35-a7f8-aa2d4b6c73cc\" }"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:29.991622841Z","hostname":"buskin","pid":1671,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:91","prefix":"Ipv6Prefix { addr: fd00:1122:3344:2::, len: 64 }"} {"msg":"Ensuring address fd00:1122:3344:3::1 exists","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:30.038585772Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2002"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:3::1, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:30.038643624Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_2698336d-5a4b-4958-8e5d-8a26f208fcce","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:30.062691602Z","hostname":"buskin","pid":1671,"sled_id":"cc74b0a2-490e-4424-adad-bb6a277ade47","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"cc74b0a2-490e-4424-adad-bb6a277ade47"} {"msg":"Failed to get switch ID from MGS (retrying in 132.25493ms)","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:30.068464954Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:379","error":"Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b7bc13c3-b6ae-4383-ad03-40766cff6ddf\", \"content-length\": \"152\", \"date\": \"Fri, 01 Mar 2024 23:15:29 GMT\"}; value: Error { error_code: Some(\"DiscoveryNotYetComplete\"), message: \"discovery process not yet complete\", request_id: \"b7bc13c3-b6ae-4383-ad03-40766cff6ddf\" }"} {"msg":"Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:30.202382475Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:408"} {"msg":"Building Port Configuration","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:30.234099672Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:567"} {"msg":"Checking dendrite uptime","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:30.234181894Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:609"} {"msg":"Dendrite online","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:30.2352458Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:612","uptime":"3"} {"msg":"Configuring default uplink on switch","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:30.23526126Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:430","config":"PortSettings {\n links: {\n \"0\": LinkSettings {\n addrs: [\n 10.151.1.174,\n ],\n params: LinkCreate {\n autoneg: false,\n fec: None,\n kr: false,\n lane: Some(\n LinkId(\n 0,\n ),\n ),\n speed: Speed40G,\n },\n },\n },\n v4_routes: {},\n v6_routes: {},\n}"} {"msg":"Received DisksChanged({})","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:30.314900391Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Ensuring address fd00:1122:3344:3::1 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:30.585234773Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2010"} {"msg":"Zone not using gateway (even though underlay is up)","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:30.585281314Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2034"} {"msg":"Setting up internal-dns service","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:31.212760885Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2187"} {"msg":"Ensuring address fd00:1122:3344:1::1 exists","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:31.789493273Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2002"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:1::1, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:31.789556315Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_720f4076-e581-438f-9cb1-67dafb6cb05c","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"Ensuring address fd00:1122:3344:1::1 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:32.341399893Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2010"} {"msg":"Zone not using gateway (even though underlay is up)","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:32.341444954Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2034"} {"msg":"Setting up internal-dns service","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:33.004325882Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2187"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:33.483340416Z","hostname":"buskin","pid":1671,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:91","prefix":"Ipv6Prefix { addr: fd00:1122:3344:3::, len: 64 }"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:33.574395517Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_internal_dns_2698336d-5a4b-4958-8e5d-8a26f208fcce"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:33.574439098Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_internal_dns_720f4076-e581-438f-9cb1-67dafb6cb05c"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:33.574446708Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_internal_dns_0fb91602-87a5-4c5e-baa6-f0c1b28cef23"} {"msg":"Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:33.574482859Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:198"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:33.57454365Z","hostname":"buskin","pid":1671,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:91","prefix":"Ipv6Prefix { addr: fd00:1122:3344:1::, len: 64 }"} {"msg":"Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:33.574760306Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:198"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:33.575059223Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"01dcc984-8afa-4a08-a748-bbc0480be84d","remote_addr":"[fd00:1122:3344:101::1]:43477","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":14044946,"response_code":"204"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:33.576643642Z","hostname":"buskin","pid":1671,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:417"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:33.690830469Z","hostname":"buskin","pid":1671,"sled_id":"cc74b0a2-490e-4424-adad-bb6a277ade47","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"cc74b0a2-490e-4424-adad-bb6a277ade47"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:33.694002307Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:37120"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:33.694203262Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"9885c20d-e469-4337-a343-0859d9ba6d3a","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:37120","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":85,"response_code":"200"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:34.185904036Z","hostname":"buskin","pid":1671,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:417"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:34.245970877Z","hostname":"buskin","pid":1671,"dns_config_addr":"[fd00:1122:3344:3::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:417"} {"msg":"Configured all DNS servers","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:34.311414591Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:453"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:34.311424371Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:254"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:34.34239167Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:271","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:34.344083411Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:296"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:34.344089721Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:188","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: 720f4076-e581-438f-9cb1-67dafb6cb05c, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 1c292cfd-336a-4f75-8eb1-9f4fab725484, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bb8fa3f4-707c-4207-88cf-a0bf61d629c1, kind: Service { id: 1c292cfd-336a-4f75-8eb1-9f4fab725484 }, name: Name(\"ntp-1c292cfd-336a-4f75-8eb1-9f4fab725484\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 148, 132])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:34.372915977Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:34.373085442Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:34434"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 720f4076-e581-438f-9cb1-67dafb6cb05c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:34.373284066Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 720f4076-e581-438f-9cb1-67dafb6cb05c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:34.373296396Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:34.394598478Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:34.3946595Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:34.412548288Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:34.414968157Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:34.429805531Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:34.429861282Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:34.430072307Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:34.43018457Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:34.441461217Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:254"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:34.47427971Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:271","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:34.475025299Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:296"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:34.475031249Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:188","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:34.520020281Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte0\", ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 148, 132]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 }, vnic: \"vopte0\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:34.520056642Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1271","zone_type":"ntp"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.1.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:34.520521923Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:98"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.1.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:34.520527903Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:105"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.1.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:34.520751909Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:133"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:34.522058101Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Configuring new Omicron zone: oxz_ntp_1c292cfd-336a-4f75-8eb1-9f4fab725484","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:34.545746401Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_ntp_1c292cfd-336a-4f75-8eb1-9f4fab725484","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:34.563886445Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:35.519093695Z","hostname":"buskin","pid":1671,"sled_id":"cc74b0a2-490e-4424-adad-bb6a277ade47","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"cc74b0a2-490e-4424-adad-bb6a277ade47"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:35.6184962Z","hostname":"buskin","pid":1671,"zone":"oxz_ntp_1c292cfd-336a-4f75-8eb1-9f4fab725484","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_ntp_1c292cfd-336a-4f75-8eb1-9f4fab725484\") failed: Property not found. retry in 62.318594ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:35.753773324Z","hostname":"buskin","pid":1671,"zone":"oxz_ntp_1c292cfd-336a-4f75-8eb1-9f4fab725484","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_1c292cfd-336a-4f75-8eb1-9f4fab725484\") failed: Property not found. retry in 57.308989ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:35.835574468Z","hostname":"buskin","pid":1671,"zone":"oxz_ntp_1c292cfd-336a-4f75-8eb1-9f4fab725484","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_1c292cfd-336a-4f75-8eb1-9f4fab725484\") failed: Property not found. retry in 203.609995ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:35.91406482Z","hostname":"buskin","pid":1671,"zone":"oxz_ntp_1c292cfd-336a-4f75-8eb1-9f4fab725484","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_1c292cfd-336a-4f75-8eb1-9f4fab725484\") failed: Property not found. retry in 344.567577ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:36.128182235Z","hostname":"buskin","pid":1671,"zone":"oxz_ntp_1c292cfd-336a-4f75-8eb1-9f4fab725484","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_1c292cfd-336a-4f75-8eb1-9f4fab725484\") failed: Property not found. retry in 413.585844ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:36.481806788Z","hostname":"buskin","pid":1671,"zone":"oxz_ntp_1c292cfd-336a-4f75-8eb1-9f4fab725484","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_1c292cfd-336a-4f75-8eb1-9f4fab725484\") failed: Property not found. retry in 1.016794464s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:36.904644606Z","hostname":"buskin","pid":1671,"zone":"oxz_ntp_1c292cfd-336a-4f75-8eb1-9f4fab725484","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_1c292cfd-336a-4f75-8eb1-9f4fab725484\") failed: Property not found. retry in 501.047767ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:37.932092495Z","hostname":"buskin","pid":1671,"zone":"oxz_ntp_1c292cfd-336a-4f75-8eb1-9f4fab725484","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_1c292cfd-336a-4f75-8eb1-9f4fab725484\") failed: Property not found. retry in 1.430295638s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:38.442373875Z","hostname":"buskin","pid":1671,"zone":"oxz_ntp_1c292cfd-336a-4f75-8eb1-9f4fab725484","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_1c292cfd-336a-4f75-8eb1-9f4fab725484\") failed: Property not found. retry in 970.345649ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:39.882988915Z","hostname":"buskin","pid":1671,"zone":"oxz_ntp_1c292cfd-336a-4f75-8eb1-9f4fab725484","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_1c292cfd-336a-4f75-8eb1-9f4fab725484\") failed: Property not found. retry in 1.369312904s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:40.864000996Z","hostname":"buskin","pid":1671,"zone":"oxz_ntp_1c292cfd-336a-4f75-8eb1-9f4fab725484","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring address fd00:1122:3344:101::1b exists","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:42.27337055Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2002"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::1b, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:42.273436011Z","hostname":"buskin","pid":1671,"zone":"oxz_ntp_1c292cfd-336a-4f75-8eb1-9f4fab725484","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"Ensuring address fd00:1122:3344:101::1b exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:42.819424446Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2010"} {"msg":"Zone using sled underlay as gateway","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:42.819473208Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2029"} {"msg":"Set up NTP service boundary=true, Servers=[\"0.pool.ntp.org\"]","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:43.629417798Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2270"} {"msg":"Ensuring address for OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:43.703323049Z","hostname":"buskin","pid":1671,"zone":"oxz_ntp_1c292cfd-336a-4f75-8eb1-9f4fab725484","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:653"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:49.213255651Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_ntp_1c292cfd-336a-4f75-8eb1-9f4fab725484"} {"msg":"Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:49.213368004Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:198"} {"msg":"Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:49.213743613Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:198"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:49.214056381Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"7c470aed-d481-4343-8b26-3abbe4184a23","remote_addr":"[fd00:1122:3344:101::1]:34434","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":14840909,"response_code":"204"} {"msg":"Waiting for rack time synchronization","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:49.216210193Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:494"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:49.245529252Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:49.245660055Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:65256"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_1c292cfd-336a-4f75-8eb1-9f4fab725484': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:49.255107046Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3393"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:49.255218409Z","hostname":"buskin","pid":1671,"uri":"/timesync","method":"GET","req_id":"22ffa4b0-5bae-4b9d-b7f6-79f7b4cd95e1","remote_addr":"[fd00:1122:3344:101::1]:65256","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":9411,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:49.256855929Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:49.407270534Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:49.407414917Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:48387"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:49.416026308Z","hostname":"buskin","pid":1671,"uri":"/timesync","method":"GET","req_id":"8b566707-655f-484e-b162-7585cc68e4a9","remote_addr":"[fd00:1122:3344:101::1]:48387","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":8581,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1709334948.1205156, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:49.41897408Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:49.418979421Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:49.550617365Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:49.550776149Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:62734"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:49.558923739Z","hostname":"buskin","pid":1671,"uri":"/timesync","method":"GET","req_id":"952a2f4d-70a9-4f54-8498-bec2ba88caa7","remote_addr":"[fd00:1122:3344:101::1]:62734","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":8097,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1709334948.1205156, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:49.560565679Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:49.560569669Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:49.705996752Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:49.706116955Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:37700"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:49.71367285Z","hostname":"buskin","pid":1671,"uri":"/timesync","method":"GET","req_id":"009cd749-e69d-40d7-8cf8-851c85099071","remote_addr":"[fd00:1122:3344:101::1]:37700","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":7510,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1709334948.1205156, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:49.716252903Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:49.716257853Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:49.983903899Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:49.984022441Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:58154"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:49.992027698Z","hostname":"buskin","pid":1671,"uri":"/timesync","method":"GET","req_id":"7de39231-4325-4e03-8de9-12233b47e3b9","remote_addr":"[fd00:1122:3344:101::1]:58154","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":7951,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1709334948.1205156, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:49.995515623Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:49.995524083Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:50.2088717Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:50.209042234Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:37667"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:50.216986708Z","hostname":"buskin","pid":1671,"uri":"/timesync","method":"GET","req_id":"60d2b46f-5942-4ee5-a69b-aeb6f5678804","remote_addr":"[fd00:1122:3344:101::1]:37667","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":7895,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1709334948.1205156, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:50.220105085Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:50.220111665Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:50.384596094Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:50.384717737Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:48351"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:50.39341659Z","hostname":"buskin","pid":1671,"uri":"/timesync","method":"GET","req_id":"613354e5-0189-4f3a-8a13-000712f3c735","remote_addr":"[fd00:1122:3344:101::1]:48351","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":8668,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1709334948.1205156, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:50.397317766Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:50.397331796Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:50.630618221Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:50.630737514Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:63496"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:50.638803682Z","hostname":"buskin","pid":1671,"uri":"/timesync","method":"GET","req_id":"da5c6de8-4c9b-4282-981a-9f24cf55daf0","remote_addr":"[fd00:1122:3344:101::1]:63496","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":8042,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1709334948.1205156, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:50.642335608Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:50.642342278Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:51.166768624Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:51.166879107Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:33773"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:51.174970495Z","hostname":"buskin","pid":1671,"uri":"/timesync","method":"GET","req_id":"0aa16b50-423a-4ccb-aa98-e1ed54e946a5","remote_addr":"[fd00:1122:3344:101::1]:33773","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":8017,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1709334948.1205156, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:51.177101287Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:51.177109598Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:51.824127057Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:51.824280231Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:43702"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:51.832972774Z","hostname":"buskin","pid":1671,"uri":"/timesync","method":"GET","req_id":"39e2f3ad-9d04-4d53-a72b-19ffce84ab48","remote_addr":"[fd00:1122:3344:101::1]:43702","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":8632,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1709334948.1205156, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:51.83527227Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:51.835278641Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:52.527439566Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:52.527588059Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:50822"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:52.535911603Z","hostname":"buskin","pid":1671,"uri":"/timesync","method":"GET","req_id":"69a4d29c-358a-4e71-aeb2-87abcf957313","remote_addr":"[fd00:1122:3344:101::1]:50822","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":8282,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1709334948.1205156, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:52.539650855Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:52.539656305Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:53.263878246Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:53.26403885Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:39664"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:53.272060036Z","hostname":"buskin","pid":1671,"uri":"/timesync","method":"GET","req_id":"73535084-f685-477d-901e-c36dee48af5b","remote_addr":"[fd00:1122:3344:101::1]:39664","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":7976,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1709334948.1205156, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:53.274744122Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:53.274750242Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:53.839910647Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:53.84004158Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:58572"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:53.848412215Z","hostname":"buskin","pid":1671,"uri":"/timesync","method":"GET","req_id":"bdc83a40-f6f6-4c95-b620-d63053275583","remote_addr":"[fd00:1122:3344:101::1]:58572","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":8346,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1709334948.1205156, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:53.851804538Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:53.851810859Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:54.349952941Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:54.350058303Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:56395"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:55.002132223Z","hostname":"buskin","pid":1671,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:776"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:55.019887618Z","hostname":"buskin","pid":1671,"uri":"/timesync","method":"GET","req_id":"84bb1cbe-af44-4bac-baca-acd6aaf7730a","remote_addr":"[fd00:1122:3344:101::1]:56395","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":20790,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: true, ref_id: 1241950280, ip_addr: 74.6.168.72, stratum: 3, ref_time: 1709334954.585444, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:55.023408904Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Finished setting up Internal DNS and NTP","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:55.023418474Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1039"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 720f4076-e581-438f-9cb1-67dafb6cb05c, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 1c292cfd-336a-4f75-8eb1-9f4fab725484, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bb8fa3f4-707c-4207-88cf-a0bf61d629c1, kind: Service { id: 1c292cfd-336a-4f75-8eb1-9f4fab725484 }, name: Name(\"ntp-1c292cfd-336a-4f75-8eb1-9f4fab725484\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 148, 132])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3c66b1e8-6f9c-4af5-9706-40f991312a36, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 7290f55a-1449-4cf7-88e8-036470e643e8, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 669c55e0-1c27-4089-9470-6ce2af215227, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 44640223-615f-49ba-b60a-a001dba9c022, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: bcf0a56e-6585-45f9-8061-123749957160, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:55.052441745Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:55.052669641Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:49770"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 720f4076-e581-438f-9cb1-67dafb6cb05c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:55.052875356Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 720f4076-e581-438f-9cb1-67dafb6cb05c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:55.052887956Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:55.072781913Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:55.072808824Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:55.090783634Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:55.090832316Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c66b1e8-6f9c-4af5-9706-40f991312a36, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:55.108408406Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3c66b1e8-6f9c-4af5-9706-40f991312a36, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:55.108437857Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7290f55a-1449-4cf7-88e8-036470e643e8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:55.192866645Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7290f55a-1449-4cf7-88e8-036470e643e8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:55.192906326Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 669c55e0-1c27-4089-9470-6ce2af215227, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:55.269885472Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 669c55e0-1c27-4089-9470-6ce2af215227, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:55.269919973Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 44640223-615f-49ba-b60a-a001dba9c022, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:55.358979354Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 44640223-615f-49ba-b60a-a001dba9c022, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:55.359016895Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcf0a56e-6585-45f9-8061-123749957160, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:55.438998974Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bcf0a56e-6585-45f9-8061-123749957160, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:55.439041266Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:55.526829906Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:55.526864107Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:55.527079062Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:55.527213106Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:55.609132752Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:55.609193074Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:55.609201114Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:55.609206524Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:55.609212374Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_bcf0a56e-6585-45f9-8061-123749957160","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:55.703120814Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_7290f55a-1449-4cf7-88e8-036470e643e8","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:55.707585573Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_44640223-615f-49ba-b60a-a001dba9c022","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:55.711644223Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_3c66b1e8-6f9c-4af5-9706-40f991312a36","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:55.715778734Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_669c55e0-1c27-4089-9470-6ce2af215227","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:55.720558951Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_cockroachdb_bcf0a56e-6585-45f9-8061-123749957160","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:55.725982134Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_7290f55a-1449-4cf7-88e8-036470e643e8","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:55.730653148Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_44640223-615f-49ba-b60a-a001dba9c022","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:55.73520083Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_3c66b1e8-6f9c-4af5-9706-40f991312a36","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:55.739869814Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_669c55e0-1c27-4089-9470-6ce2af215227","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:55.744379565Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_cockroachdb_7290f55a-1449-4cf7-88e8-036470e643e8:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:59.079540141Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:59.079733816Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_7290f55a-1449-4cf7-88e8-036470e643e8","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:59.082952054Z","hostname":"buskin","pid":1671,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:776"} {"msg":"Profile for oxz_cockroachdb_669c55e0-1c27-4089-9470-6ce2af215227:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:59.219197762Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:59.219364996Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_669c55e0-1c27-4089-9470-6ce2af215227","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_cockroachdb_7290f55a-1449-4cf7-88e8-036470e643e8\") failed: Property not found. retry in 67.455559ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:59.233265377Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_7290f55a-1449-4cf7-88e8-036470e643e8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_cockroachdb_3c66b1e8-6f9c-4af5-9706-40f991312a36:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:59.28278766Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:59.282929783Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_3c66b1e8-6f9c-4af5-9706-40f991312a36","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_cockroachdb_7290f55a-1449-4cf7-88e8-036470e643e8\") failed: Property not found. retry in 74.824382ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:59.326442259Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_7290f55a-1449-4cf7-88e8-036470e643e8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_cockroachdb_44640223-615f-49ba-b60a-a001dba9c022:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:59.343058536Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:59.343510427Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_44640223-615f-49ba-b60a-a001dba9c022","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_cockroachdb_bcf0a56e-6585-45f9-8061-123749957160:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:59.398544335Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:15:59.39911728Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_bcf0a56e-6585-45f9-8061-123749957160","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_cockroachdb_7290f55a-1449-4cf7-88e8-036470e643e8\") failed: Property not found. retry in 292.791984ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:59.429415562Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_7290f55a-1449-4cf7-88e8-036470e643e8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_669c55e0-1c27-4089-9470-6ce2af215227\") failed: Property not found. retry in 38.493307ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:59.470860407Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_669c55e0-1c27-4089-9470-6ce2af215227","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_669c55e0-1c27-4089-9470-6ce2af215227\") failed: Property not found. retry in 94.366243ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:59.545834014Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_669c55e0-1c27-4089-9470-6ce2af215227","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_669c55e0-1c27-4089-9470-6ce2af215227\") failed: Property not found. retry in 277.292213ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:59.676407392Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_669c55e0-1c27-4089-9470-6ce2af215227","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3c66b1e8-6f9c-4af5-9706-40f991312a36\") failed: Property not found. retry in 69.689117ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:59.718949204Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_3c66b1e8-6f9c-4af5-9706-40f991312a36","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_44640223-615f-49ba-b60a-a001dba9c022\") failed: Property not found. retry in 51.906753ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:59.737035527Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_44640223-615f-49ba-b60a-a001dba9c022","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_7290f55a-1449-4cf7-88e8-036470e643e8\") failed: Property not found. retry in 287.135051ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:59.759300023Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_7290f55a-1449-4cf7-88e8-036470e643e8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bcf0a56e-6585-45f9-8061-123749957160\") failed: Property not found. retry in 36.9686ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:59.834906365Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_bcf0a56e-6585-45f9-8061-123749957160","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_44640223-615f-49ba-b60a-a001dba9c022\") failed: Property not found. retry in 54.136861ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:15:59.917879737Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_44640223-615f-49ba-b60a-a001dba9c022","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3c66b1e8-6f9c-4af5-9706-40f991312a36\") failed: Property not found. retry in 61.6542ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:00.007244246Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_3c66b1e8-6f9c-4af5-9706-40f991312a36","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_669c55e0-1c27-4089-9470-6ce2af215227\") failed: Property not found. retry in 313.600187ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:00.135054668Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_669c55e0-1c27-4089-9470-6ce2af215227","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bcf0a56e-6585-45f9-8061-123749957160\") failed: Property not found. retry in 100.672466ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:00.212932985Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_bcf0a56e-6585-45f9-8061-123749957160","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_7290f55a-1449-4cf7-88e8-036470e643e8\") failed: Property not found. retry in 835.492063ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:00.280131442Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_7290f55a-1449-4cf7-88e8-036470e643e8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_44640223-615f-49ba-b60a-a001dba9c022\") failed: Property not found. retry in 264.193955ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:00.383425922Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_44640223-615f-49ba-b60a-a001dba9c022","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3c66b1e8-6f9c-4af5-9706-40f991312a36\") failed: Property not found. retry in 182.208485ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:00.420923711Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_3c66b1e8-6f9c-4af5-9706-40f991312a36","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bcf0a56e-6585-45f9-8061-123749957160\") failed: Property not found. retry in 242.417087ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:00.448213289Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_bcf0a56e-6585-45f9-8061-123749957160","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_669c55e0-1c27-4089-9470-6ce2af215227\") failed: Property not found. retry in 841.04037ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:00.470757521Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_669c55e0-1c27-4089-9470-6ce2af215227","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3c66b1e8-6f9c-4af5-9706-40f991312a36\") failed: Property not found. retry in 286.360228ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:00.689061878Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_3c66b1e8-6f9c-4af5-9706-40f991312a36","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_44640223-615f-49ba-b60a-a001dba9c022\") failed: Property not found. retry in 410.230815ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:00.763874771Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_44640223-615f-49ba-b60a-a001dba9c022","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bcf0a56e-6585-45f9-8061-123749957160\") failed: Property not found. retry in 559.405713ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:00.807020138Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_bcf0a56e-6585-45f9-8061-123749957160","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3c66b1e8-6f9c-4af5-9706-40f991312a36\") failed: Property not found. retry in 1.046649503s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:01.019577515Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_3c66b1e8-6f9c-4af5-9706-40f991312a36","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_7290f55a-1449-4cf7-88e8-036470e643e8\") failed: Property not found. retry in 1.148137135s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:01.149006545Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_7290f55a-1449-4cf7-88e8-036470e643e8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_44640223-615f-49ba-b60a-a001dba9c022\") failed: Property not found. retry in 953.93545ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:01.218774594Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_44640223-615f-49ba-b60a-a001dba9c022","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_669c55e0-1c27-4089-9470-6ce2af215227\") failed: Property not found. retry in 1.200577463s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:01.339567363Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_669c55e0-1c27-4089-9470-6ce2af215227","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bcf0a56e-6585-45f9-8061-123749957160\") failed: Property not found. retry in 1.11409336s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:01.395516134Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_bcf0a56e-6585-45f9-8061-123749957160","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3c66b1e8-6f9c-4af5-9706-40f991312a36\") failed: Property not found. retry in 934.143014ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:02.077359386Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_3c66b1e8-6f9c-4af5-9706-40f991312a36","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_44640223-615f-49ba-b60a-a001dba9c022\") failed: Property not found. retry in 878.618946ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:02.183979988Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_44640223-615f-49ba-b60a-a001dba9c022","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_7290f55a-1449-4cf7-88e8-036470e643e8\") failed: Property not found. retry in 747.235009ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:02.307377801Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_7290f55a-1449-4cf7-88e8-036470e643e8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bcf0a56e-6585-45f9-8061-123749957160\") failed: Property not found. retry in 976.965626ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:02.51840232Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_bcf0a56e-6585-45f9-8061-123749957160","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_669c55e0-1c27-4089-9470-6ce2af215227\") failed: Property not found. retry in 556.135383ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:02.549351909Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_669c55e0-1c27-4089-9470-6ce2af215227","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3c66b1e8-6f9c-4af5-9706-40f991312a36\") failed: Property not found. retry in 1.45856906s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:03.021495064Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_3c66b1e8-6f9c-4af5-9706-40f991312a36","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_7290f55a-1449-4cf7-88e8-036470e643e8\") failed: Property not found. retry in 1.070892264s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:03.063637526Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_7290f55a-1449-4cf7-88e8-036470e643e8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_44640223-615f-49ba-b60a-a001dba9c022\") failed: Property not found. retry in 1.322761265s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:03.071707754Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_44640223-615f-49ba-b60a-a001dba9c022","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_669c55e0-1c27-4089-9470-6ce2af215227\") failed: Property not found. retry in 751.48569ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:03.114197015Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_669c55e0-1c27-4089-9470-6ce2af215227","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bcf0a56e-6585-45f9-8061-123749957160\") failed: Property not found. retry in 718.23211ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:03.504718941Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_bcf0a56e-6585-45f9-8061-123749957160","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_669c55e0-1c27-4089-9470-6ce2af215227\") failed: Property not found. retry in 929.287678ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:03.874439668Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_669c55e0-1c27-4089-9470-6ce2af215227","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_7290f55a-1449-4cf7-88e8-036470e643e8\") failed: Property not found. retry in 763.76676ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:04.142493944Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_7290f55a-1449-4cf7-88e8-036470e643e8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bcf0a56e-6585-45f9-8061-123749957160\") failed: Property not found. retry in 581.552733ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:04.231645058Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_bcf0a56e-6585-45f9-8061-123749957160","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:04.346914942Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:33377"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:04.347112127Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"bddd90aa-7e83-438f-badb-cc26563d80c0","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:33377","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":109,"response_code":"200"} {"msg":"wait for service Some(\"oxz_cockroachdb_44640223-615f-49ba-b60a-a001dba9c022\") failed: Property not found. retry in 629.117907ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:04.403758834Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_44640223-615f-49ba-b60a-a001dba9c022","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3c66b1e8-6f9c-4af5-9706-40f991312a36\") failed: Property not found. retry in 1.00441778s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:04.489072014Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_3c66b1e8-6f9c-4af5-9706-40f991312a36","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_669c55e0-1c27-4089-9470-6ce2af215227\") failed: Property not found. retry in 1.424571174s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:04.811961644Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_669c55e0-1c27-4089-9470-6ce2af215227","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bcf0a56e-6585-45f9-8061-123749957160\") failed: Property not found. retry in 887.795242ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:04.821497057Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_bcf0a56e-6585-45f9-8061-123749957160","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_7290f55a-1449-4cf7-88e8-036470e643e8\") failed: Property not found. retry in 566.678766ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:04.915068529Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_7290f55a-1449-4cf7-88e8-036470e643e8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_44640223-615f-49ba-b60a-a001dba9c022\") failed: Property not found. retry in 1.198594257s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:05.041820974Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_44640223-615f-49ba-b60a-a001dba9c022","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_7290f55a-1449-4cf7-88e8-036470e643e8\") failed: Property not found. retry in 732.620279ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:05.490089606Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_7290f55a-1449-4cf7-88e8-036470e643e8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3c66b1e8-6f9c-4af5-9706-40f991312a36\") failed: Property not found. retry in 1.226385213s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:05.502703175Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_3c66b1e8-6f9c-4af5-9706-40f991312a36","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bcf0a56e-6585-45f9-8061-123749957160\") failed: Property not found. retry in 1.23818282s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:05.716639815Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_bcf0a56e-6585-45f9-8061-123749957160","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_7290f55a-1449-4cf7-88e8-036470e643e8\") failed: Property not found. retry in 1.326104505s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:06.231976369Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_7290f55a-1449-4cf7-88e8-036470e643e8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_669c55e0-1c27-4089-9470-6ce2af215227\") failed: Property not found. retry in 1.312305719s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:06.245262714Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_669c55e0-1c27-4089-9470-6ce2af215227","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_44640223-615f-49ba-b60a-a001dba9c022\") failed: Property not found. retry in 536.713509ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:06.253039525Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_44640223-615f-49ba-b60a-a001dba9c022","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3c66b1e8-6f9c-4af5-9706-40f991312a36\") failed: Property not found. retry in 533.707977ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:06.739313946Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_3c66b1e8-6f9c-4af5-9706-40f991312a36","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_44640223-615f-49ba-b60a-a001dba9c022\") failed: Property not found. retry in 1.110232472s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:06.799414108Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_44640223-615f-49ba-b60a-a001dba9c022","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bcf0a56e-6585-45f9-8061-123749957160\") failed: Property not found. retry in 1.045587445s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:06.963716193Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_bcf0a56e-6585-45f9-8061-123749957160","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3c66b1e8-6f9c-4af5-9706-40f991312a36\") failed: Property not found. retry in 1.489303796s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:07.283145778Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_3c66b1e8-6f9c-4af5-9706-40f991312a36","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_7290f55a-1449-4cf7-88e8-036470e643e8\") failed: Property not found. retry in 1.439138736s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:07.569195496Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_7290f55a-1449-4cf7-88e8-036470e643e8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_669c55e0-1c27-4089-9470-6ce2af215227\") failed: Property not found. retry in 645.362024ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:07.588028796Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_669c55e0-1c27-4089-9470-6ce2af215227","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_44640223-615f-49ba-b60a-a001dba9c022\") failed: Property not found. retry in 761.76305ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:07.922869429Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_44640223-615f-49ba-b60a-a001dba9c022","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bcf0a56e-6585-45f9-8061-123749957160\") failed: Property not found. retry in 1.445187528s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:08.020947191Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_bcf0a56e-6585-45f9-8061-123749957160","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_669c55e0-1c27-4089-9470-6ce2af215227\") failed: Property not found. retry in 1.427607564s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:08.244946269Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_669c55e0-1c27-4089-9470-6ce2af215227","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_44640223-615f-49ba-b60a-a001dba9c022\") failed: Property not found. retry in 909.744613ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:08.694840439Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_44640223-615f-49ba-b60a-a001dba9c022","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3c66b1e8-6f9c-4af5-9706-40f991312a36\") failed: Property not found. retry in 1.215500192s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:08.781662766Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_3c66b1e8-6f9c-4af5-9706-40f991312a36","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_7290f55a-1449-4cf7-88e8-036470e643e8\") failed: Property not found. retry in 680.732231ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:09.016264883Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_7290f55a-1449-4cf7-88e8-036470e643e8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bcf0a56e-6585-45f9-8061-123749957160\") failed: Property not found. retry in 1.33254565s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:09.474675802Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_bcf0a56e-6585-45f9-8061-123749957160","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_44640223-615f-49ba-b60a-a001dba9c022\") failed: Property not found. retry in 622.868907ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:09.612875567Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_44640223-615f-49ba-b60a-a001dba9c022","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_669c55e0-1c27-4089-9470-6ce2af215227\") failed: Property not found. retry in 1.122403701s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:09.680885753Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_669c55e0-1c27-4089-9470-6ce2af215227","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_7290f55a-1449-4cf7-88e8-036470e643e8\") failed: Property not found. retry in 1.238246742s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:09.706670985Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_7290f55a-1449-4cf7-88e8-036470e643e8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3c66b1e8-6f9c-4af5-9706-40f991312a36\") failed: Property not found. retry in 1.326846194s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:10.007270608Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_3c66b1e8-6f9c-4af5-9706-40f991312a36","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_44640223-615f-49ba-b60a-a001dba9c022\") failed: Property not found. retry in 1.423436613s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:10.24614984Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_44640223-615f-49ba-b60a-a001dba9c022","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_669c55e0-1c27-4089-9470-6ce2af215227\") failed: Property not found. retry in 1.055392849s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:10.812965024Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_669c55e0-1c27-4089-9470-6ce2af215227","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bcf0a56e-6585-45f9-8061-123749957160\") failed: Property not found. retry in 1.086312432s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:10.820263643Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_bcf0a56e-6585-45f9-8061-123749957160","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_7290f55a-1449-4cf7-88e8-036470e643e8\") failed: Property not found. retry in 1.148374899s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:10.954623545Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_7290f55a-1449-4cf7-88e8-036470e643e8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bcf0a56e-6585-45f9-8061-123749957160\") failed: Property not found. retry in 882.202425ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:11.91539121Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_bcf0a56e-6585-45f9-8061-123749957160","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bcf0a56e-6585-45f9-8061-123749957160\") failed: Property not found. retry in 800.518598ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:12.808343014Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_bcf0a56e-6585-45f9-8061-123749957160","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bcf0a56e-6585-45f9-8061-123749957160\") failed: Property not found. retry in 661.18144ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:13.618970831Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_bcf0a56e-6585-45f9-8061-123749957160","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bcf0a56e-6585-45f9-8061-123749957160\") failed: Property not found. retry in 687.911891ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:14.289825245Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_bcf0a56e-6585-45f9-8061-123749957160","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bcf0a56e-6585-45f9-8061-123749957160\") failed: Property not found. retry in 868.408319ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:14.989079264Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_bcf0a56e-6585-45f9-8061-123749957160","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bcf0a56e-6585-45f9-8061-123749957160\") failed: Property not found. retry in 683.911208ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:15.869715316Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_bcf0a56e-6585-45f9-8061-123749957160","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bcf0a56e-6585-45f9-8061-123749957160\") failed: Property not found. retry in 992.809978ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:16.563468231Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_bcf0a56e-6585-45f9-8061-123749957160","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bcf0a56e-6585-45f9-8061-123749957160\") failed: Property not found. retry in 1.281918547s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:17.565820015Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_bcf0a56e-6585-45f9-8061-123749957160","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bcf0a56e-6585-45f9-8061-123749957160\") failed: Property not found. retry in 520.436108ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:18.857570448Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_bcf0a56e-6585-45f9-8061-123749957160","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bcf0a56e-6585-45f9-8061-123749957160\") failed: Property not found. retry in 1.468257239s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:19.392614225Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_bcf0a56e-6585-45f9-8061-123749957160","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bcf0a56e-6585-45f9-8061-123749957160\") failed: Property not found. retry in 1.341369676s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:20.871761948Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_bcf0a56e-6585-45f9-8061-123749957160","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:22.235456563Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_cockroachdb_bcf0a56e-6585-45f9-8061-123749957160"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:22.235520115Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_cockroachdb_7290f55a-1449-4cf7-88e8-036470e643e8"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:22.235529375Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_cockroachdb_44640223-615f-49ba-b60a-a001dba9c022"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:22.235538035Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_cockroachdb_3c66b1e8-6f9c-4af5-9706-40f991312a36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:22.235544155Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_cockroachdb_669c55e0-1c27-4089-9470-6ce2af215227"} {"msg":"Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:22.235601297Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:198"} {"msg":"Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:22.235910954Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:198"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:22.236244452Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"7c396ea2-a8b9-46f6-b137-6686f839bde2","remote_addr":"[fd00:1122:3344:101::1]:49770","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":27213496,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:22.269078497Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:38967"} {"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::3]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:22.297976775Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3227"} {"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:22.487068857Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3245"} {"msg":"Formatting CRDB - Completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:28.985565565Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3268"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:28.985888373Z","hostname":"buskin","pid":1671,"uri":"/cockroachdb","method":"POST","req_id":"f51f35f1-3a0f-403e-904a-6dc387bb6967","remote_addr":"[fd00:1122:3344:101::1]:38967","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":6716721,"response_code":"204"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 720f4076-e581-438f-9cb1-67dafb6cb05c, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 1c292cfd-336a-4f75-8eb1-9f4fab725484, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bb8fa3f4-707c-4207-88cf-a0bf61d629c1, kind: Service { id: 1c292cfd-336a-4f75-8eb1-9f4fab725484 }, name: Name(\"ntp-1c292cfd-336a-4f75-8eb1-9f4fab725484\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 148, 132])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3c66b1e8-6f9c-4af5-9706-40f991312a36, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 7290f55a-1449-4cf7-88e8-036470e643e8, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 669c55e0-1c27-4089-9470-6ce2af215227, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 44640223-615f-49ba-b60a-a001dba9c022, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: bcf0a56e-6585-45f9-8061-123749957160, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 0a603215-e206-4aa0-8930-250f278a0d89, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 3af2d634-ca96-4144-9913-082ef495b783, kind: Service { id: 0a603215-e206-4aa0-8930-250f278a0d89 }, name: Name(\"external-dns-0a603215-e206-4aa0-8930-250f278a0d89\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 224, 81])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 3f949516-3930-421a-bae2-b58ff4ea0dc8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: d086f287-39ab-42aa-b1b5-4d91b0388531, kind: Service { id: 3f949516-3930-421a-bae2-b58ff4ea0dc8 }, name: Name(\"external-dns-3f949516-3930-421a-bae2-b58ff4ea0dc8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 146])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 4ea96463-fca7-4b4a-8a70-4e88cc368897, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: c6ce2abe-65af-4c28-bdee-3dca5bb68c72, kind: Service { id: 4ea96463-fca7-4b4a-8a70-4e88cc368897 }, name: Name(\"nexus-4ea96463-fca7-4b4a-8a70-4e88cc368897\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 933c0d3a-d49d-431b-8f65-51222f67e8f3, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 06ae0dac-13b1-4ef0-995e-b56e4a7d3138, kind: Service { id: 933c0d3a-d49d-431b-8f65-51222f67e8f3 }, name: Name(\"nexus-933c0d3a-d49d-431b-8f65-51222f67e8f3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 46])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 264d5dd4-3eb2-4adb-b639-f0a26ab5534d, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 9eeb19bd-897d-4d56-ae7e-bd0911ac9557, kind: Service { id: 264d5dd4-3eb2-4adb-b639-f0a26ab5534d }, name: Name(\"nexus-264d5dd4-3eb2-4adb-b639-f0a26ab5534d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 168, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: a87cb16a-062c-4865-a37d-8e0d7e2bd2b9, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 74802856-0b1c-433c-9e82-d4aedb5e0749, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ecba71f7-525f-4818-b686-c9ff63ca561f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 925e71e3-e0b8-4973-8ff8-5438a755ef84, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 282e3323-14c4-4d62-8807-8693fcbf40ea, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 3de24d1d-6129-402e-99c6-cbe1a3727280, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 7f367ce2-eb70-48d4-b9f5-2554c178c7d8, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: cf8e7366-58a5-4520-b2ad-904e641779d8, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 81f36664-5b2e-47bc-8f86-8336a864a7bb, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 5bbf29d8-d45a-494e-b486-773cde91df47, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 6c91a708-fa2b-487f-8528-4d94d9438931, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 85a1d6ba-7262-4d27-a9fc-1d1499e79005, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 056f8fb6-5500-4b9e-92d0-2be6e076a7d3, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 5fb374a2-2d09-404e-b2d8-6f14e3884eab, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:29.042125001Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:29.042868279Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:42557"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 720f4076-e581-438f-9cb1-67dafb6cb05c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:29.043345781Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 720f4076-e581-438f-9cb1-67dafb6cb05c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:29.043365301Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:29.085096504Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:29.085175536Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:29.117862956Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:29.117945688Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c66b1e8-6f9c-4af5-9706-40f991312a36, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:29.146517278Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3c66b1e8-6f9c-4af5-9706-40f991312a36, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:29.14659602Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7290f55a-1449-4cf7-88e8-036470e643e8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:29.174871693Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7290f55a-1449-4cf7-88e8-036470e643e8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:29.174939255Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 669c55e0-1c27-4089-9470-6ce2af215227, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:29.201666579Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 669c55e0-1c27-4089-9470-6ce2af215227, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:29.201732271Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 44640223-615f-49ba-b60a-a001dba9c022, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:29.227498202Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 44640223-615f-49ba-b60a-a001dba9c022, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:29.227583244Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcf0a56e-6585-45f9-8061-123749957160, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:29.256423661Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bcf0a56e-6585-45f9-8061-123749957160, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:29.256516113Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a603215-e206-4aa0-8930-250f278a0d89, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:29.288301201Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0a603215-e206-4aa0-8930-250f278a0d89, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:29.288365303Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f949516-3930-421a-bae2-b58ff4ea0dc8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:29.551802296Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3f949516-3930-421a-bae2-b58ff4ea0dc8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:29.551899769Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 74802856-0b1c-433c-9e82-d4aedb5e0749, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:29.649245893Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 74802856-0b1c-433c-9e82-d4aedb5e0749, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:29.649331545Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3de24d1d-6129-402e-99c6-cbe1a3727280, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:29.743058471Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3de24d1d-6129-402e-99c6-cbe1a3727280, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:29.743151023Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f367ce2-eb70-48d4-b9f5-2554c178c7d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:29.838216002Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7f367ce2-eb70-48d4-b9f5-2554c178c7d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:29.838305194Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cf8e7366-58a5-4520-b2ad-904e641779d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:29.925805557Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cf8e7366-58a5-4520-b2ad-904e641779d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:29.925886959Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 81f36664-5b2e-47bc-8f86-8336a864a7bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:30.015837753Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 81f36664-5b2e-47bc-8f86-8336a864a7bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:30.015913125Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5bbf29d8-d45a-494e-b486-773cde91df47, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:30.099379349Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5bbf29d8-d45a-494e-b486-773cde91df47, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:30.099462112Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c91a708-fa2b-487f-8528-4d94d9438931, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:30.187499688Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6c91a708-fa2b-487f-8528-4d94d9438931, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:30.18758356Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85a1d6ba-7262-4d27-a9fc-1d1499e79005, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:30.267668262Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 85a1d6ba-7262-4d27-a9fc-1d1499e79005, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:30.267749234Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 056f8fb6-5500-4b9e-92d0-2be6e076a7d3, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:30.78316225Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 056f8fb6-5500-4b9e-92d0-2be6e076a7d3, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:30.787183378Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5fb374a2-2d09-404e-b2d8-6f14e3884eab, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:30.86032141Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5fb374a2-2d09-404e-b2d8-6f14e3884eab, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:30.860388791Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:30.938077684Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:30.938165367Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:30.938480924Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:30.93869416Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:31.074145888Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:31.074209139Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:31.074217949Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:31.0742262Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:31.07423436Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:31.0742542Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:31.07426215Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:31.074269611Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:31.074275741Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:31.074281451Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:31.074288061Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:31.074295691Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:31.074301262Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:31.074306831Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:31.074313142Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:31.074318522Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:31.074332642Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:31.074338222Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:31.074343443Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:31.074498386Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"9bca79ec-08c7-4fdd-8a09-a55642b4b0c2","remote_addr":"[fd00:1122:3344:101::1]:42557","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea\", TimeNotSynchronized), (\"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47\", TimeNotSynchronized), (\"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8\", TimeNotSynchronized), (\"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89\", TimeNotSynchronized), (\"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749\", TimeNotSynchronized), (\"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8\", TimeNotSynchronized), (\"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3\", TimeNotSynchronized), (\"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84\", TimeNotSynchronized), (\"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb\", TimeNotSynchronized), (\"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005\", TimeNotSynchronized), (\"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3\", TimeNotSynchronized), (\"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab\", TimeNotSynchronized), (\"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280\", TimeNotSynchronized), (\"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897\", TimeNotSynchronized), (\"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931\", TimeNotSynchronized), (\"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8\", TimeNotSynchronized), (\"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d\", TimeNotSynchronized), (\"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9\", TimeNotSynchronized), (\"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f\", TimeNotSynchronized)]","latency_us":2031474,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:31.074858695Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"75.209938ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9bca79ec-08c7-4fdd-8a09-a55642b4b0c2\", \"content-length\": \"124\", \"date\": \"Fri, 01 Mar 2024 23:16:31 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"9bca79ec-08c7-4fdd-8a09-a55642b4b0c2\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 720f4076-e581-438f-9cb1-67dafb6cb05c, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 1c292cfd-336a-4f75-8eb1-9f4fab725484, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bb8fa3f4-707c-4207-88cf-a0bf61d629c1, kind: Service { id: 1c292cfd-336a-4f75-8eb1-9f4fab725484 }, name: Name(\"ntp-1c292cfd-336a-4f75-8eb1-9f4fab725484\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 148, 132])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3c66b1e8-6f9c-4af5-9706-40f991312a36, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 7290f55a-1449-4cf7-88e8-036470e643e8, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 669c55e0-1c27-4089-9470-6ce2af215227, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 44640223-615f-49ba-b60a-a001dba9c022, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: bcf0a56e-6585-45f9-8061-123749957160, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 0a603215-e206-4aa0-8930-250f278a0d89, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 3af2d634-ca96-4144-9913-082ef495b783, kind: Service { id: 0a603215-e206-4aa0-8930-250f278a0d89 }, name: Name(\"external-dns-0a603215-e206-4aa0-8930-250f278a0d89\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 224, 81])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 3f949516-3930-421a-bae2-b58ff4ea0dc8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: d086f287-39ab-42aa-b1b5-4d91b0388531, kind: Service { id: 3f949516-3930-421a-bae2-b58ff4ea0dc8 }, name: Name(\"external-dns-3f949516-3930-421a-bae2-b58ff4ea0dc8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 146])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 4ea96463-fca7-4b4a-8a70-4e88cc368897, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: c6ce2abe-65af-4c28-bdee-3dca5bb68c72, kind: Service { id: 4ea96463-fca7-4b4a-8a70-4e88cc368897 }, name: Name(\"nexus-4ea96463-fca7-4b4a-8a70-4e88cc368897\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 933c0d3a-d49d-431b-8f65-51222f67e8f3, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 06ae0dac-13b1-4ef0-995e-b56e4a7d3138, kind: Service { id: 933c0d3a-d49d-431b-8f65-51222f67e8f3 }, name: Name(\"nexus-933c0d3a-d49d-431b-8f65-51222f67e8f3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 46])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 264d5dd4-3eb2-4adb-b639-f0a26ab5534d, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 9eeb19bd-897d-4d56-ae7e-bd0911ac9557, kind: Service { id: 264d5dd4-3eb2-4adb-b639-f0a26ab5534d }, name: Name(\"nexus-264d5dd4-3eb2-4adb-b639-f0a26ab5534d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 168, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: a87cb16a-062c-4865-a37d-8e0d7e2bd2b9, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 74802856-0b1c-433c-9e82-d4aedb5e0749, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ecba71f7-525f-4818-b686-c9ff63ca561f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 925e71e3-e0b8-4973-8ff8-5438a755ef84, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 282e3323-14c4-4d62-8807-8693fcbf40ea, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 3de24d1d-6129-402e-99c6-cbe1a3727280, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 7f367ce2-eb70-48d4-b9f5-2554c178c7d8, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: cf8e7366-58a5-4520-b2ad-904e641779d8, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 81f36664-5b2e-47bc-8f86-8336a864a7bb, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 5bbf29d8-d45a-494e-b486-773cde91df47, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 6c91a708-fa2b-487f-8528-4d94d9438931, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 85a1d6ba-7262-4d27-a9fc-1d1499e79005, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 056f8fb6-5500-4b9e-92d0-2be6e076a7d3, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 5fb374a2-2d09-404e-b2d8-6f14e3884eab, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:31.151048551Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 720f4076-e581-438f-9cb1-67dafb6cb05c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:31.151946404Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 720f4076-e581-438f-9cb1-67dafb6cb05c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:31.151970584Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:31.182590964Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:31.182659596Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:31.212104577Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:31.212181089Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c66b1e8-6f9c-4af5-9706-40f991312a36, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:31.242257446Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3c66b1e8-6f9c-4af5-9706-40f991312a36, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:31.245491865Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7290f55a-1449-4cf7-88e8-036470e643e8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:31.268069828Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7290f55a-1449-4cf7-88e8-036470e643e8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:31.26814147Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 669c55e0-1c27-4089-9470-6ce2af215227, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:31.291981464Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 669c55e0-1c27-4089-9470-6ce2af215227, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:31.292054326Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 44640223-615f-49ba-b60a-a001dba9c022, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:31.318311999Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 44640223-615f-49ba-b60a-a001dba9c022, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:31.318392361Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcf0a56e-6585-45f9-8061-123749957160, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:31.344753477Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bcf0a56e-6585-45f9-8061-123749957160, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:31.344819808Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a603215-e206-4aa0-8930-250f278a0d89, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:31.371379959Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0a603215-e206-4aa0-8930-250f278a0d89, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:31.371452801Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f949516-3930-421a-bae2-b58ff4ea0dc8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:31.398590175Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3f949516-3930-421a-bae2-b58ff4ea0dc8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:31.398652227Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 74802856-0b1c-433c-9e82-d4aedb5e0749, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:31.425029353Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 74802856-0b1c-433c-9e82-d4aedb5e0749, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:31.427595126Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3de24d1d-6129-402e-99c6-cbe1a3727280, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:31.449648916Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3de24d1d-6129-402e-99c6-cbe1a3727280, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:31.449712448Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f367ce2-eb70-48d4-b9f5-2554c178c7d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:31.476522015Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7f367ce2-eb70-48d4-b9f5-2554c178c7d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:31.476580246Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cf8e7366-58a5-4520-b2ad-904e641779d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:31.504520681Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cf8e7366-58a5-4520-b2ad-904e641779d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:31.504576882Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 81f36664-5b2e-47bc-8f86-8336a864a7bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:31.533251334Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 81f36664-5b2e-47bc-8f86-8336a864a7bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:31.533339607Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5bbf29d8-d45a-494e-b486-773cde91df47, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:31.561886246Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5bbf29d8-d45a-494e-b486-773cde91df47, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:31.561966348Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c91a708-fa2b-487f-8528-4d94d9438931, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:31.591203274Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6c91a708-fa2b-487f-8528-4d94d9438931, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:31.591267995Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85a1d6ba-7262-4d27-a9fc-1d1499e79005, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:31.619804625Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 85a1d6ba-7262-4d27-a9fc-1d1499e79005, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:31.619870096Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 056f8fb6-5500-4b9e-92d0-2be6e076a7d3, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:31.646694223Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 056f8fb6-5500-4b9e-92d0-2be6e076a7d3, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:31.646768965Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5fb374a2-2d09-404e-b2d8-6f14e3884eab, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:31.67472829Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5fb374a2-2d09-404e-b2d8-6f14e3884eab, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:31.674793132Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:31.703515825Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:31.703586917Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:31.703897014Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:31.704150561Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:31.833334465Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:31.833406077Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:31.833432308Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:31.833441838Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:31.833451878Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:31.833461899Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:31.833467788Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:31.833475749Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:31.833482929Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:31.833491519Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:31.833499959Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:31.8335086Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:31.83351988Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:31.83352996Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:31.83353939Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:31.833545401Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:31.833557681Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:31.833582971Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:31.833591542Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:31.833806467Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"670f6682-502d-464a-806e-9530060eec2f","remote_addr":"[fd00:1122:3344:101::1]:42557","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8\", TimeNotSynchronized), (\"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280\", TimeNotSynchronized), (\"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8\", TimeNotSynchronized), (\"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47\", TimeNotSynchronized), (\"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749\", TimeNotSynchronized), (\"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005\", TimeNotSynchronized), (\"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d\", TimeNotSynchronized), (\"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84\", TimeNotSynchronized), (\"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931\", TimeNotSynchronized), (\"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3\", TimeNotSynchronized), (\"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab\", TimeNotSynchronized), (\"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9\", TimeNotSynchronized), (\"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3\", TimeNotSynchronized), (\"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897\", TimeNotSynchronized), (\"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89\", TimeNotSynchronized), (\"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f\", TimeNotSynchronized), (\"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea\", TimeNotSynchronized), (\"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8\", TimeNotSynchronized), (\"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb\", TimeNotSynchronized)]","latency_us":682156,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:31.834096274Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"87.101556ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"670f6682-502d-464a-806e-9530060eec2f\", \"content-length\": \"124\", \"date\": \"Fri, 01 Mar 2024 23:16:31 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"670f6682-502d-464a-806e-9530060eec2f\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 720f4076-e581-438f-9cb1-67dafb6cb05c, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 1c292cfd-336a-4f75-8eb1-9f4fab725484, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bb8fa3f4-707c-4207-88cf-a0bf61d629c1, kind: Service { id: 1c292cfd-336a-4f75-8eb1-9f4fab725484 }, name: Name(\"ntp-1c292cfd-336a-4f75-8eb1-9f4fab725484\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 148, 132])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3c66b1e8-6f9c-4af5-9706-40f991312a36, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 7290f55a-1449-4cf7-88e8-036470e643e8, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 669c55e0-1c27-4089-9470-6ce2af215227, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 44640223-615f-49ba-b60a-a001dba9c022, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: bcf0a56e-6585-45f9-8061-123749957160, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 0a603215-e206-4aa0-8930-250f278a0d89, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 3af2d634-ca96-4144-9913-082ef495b783, kind: Service { id: 0a603215-e206-4aa0-8930-250f278a0d89 }, name: Name(\"external-dns-0a603215-e206-4aa0-8930-250f278a0d89\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 224, 81])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 3f949516-3930-421a-bae2-b58ff4ea0dc8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: d086f287-39ab-42aa-b1b5-4d91b0388531, kind: Service { id: 3f949516-3930-421a-bae2-b58ff4ea0dc8 }, name: Name(\"external-dns-3f949516-3930-421a-bae2-b58ff4ea0dc8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 146])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 4ea96463-fca7-4b4a-8a70-4e88cc368897, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: c6ce2abe-65af-4c28-bdee-3dca5bb68c72, kind: Service { id: 4ea96463-fca7-4b4a-8a70-4e88cc368897 }, name: Name(\"nexus-4ea96463-fca7-4b4a-8a70-4e88cc368897\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 933c0d3a-d49d-431b-8f65-51222f67e8f3, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 06ae0dac-13b1-4ef0-995e-b56e4a7d3138, kind: Service { id: 933c0d3a-d49d-431b-8f65-51222f67e8f3 }, name: Name(\"nexus-933c0d3a-d49d-431b-8f65-51222f67e8f3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 46])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 264d5dd4-3eb2-4adb-b639-f0a26ab5534d, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 9eeb19bd-897d-4d56-ae7e-bd0911ac9557, kind: Service { id: 264d5dd4-3eb2-4adb-b639-f0a26ab5534d }, name: Name(\"nexus-264d5dd4-3eb2-4adb-b639-f0a26ab5534d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 168, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: a87cb16a-062c-4865-a37d-8e0d7e2bd2b9, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 74802856-0b1c-433c-9e82-d4aedb5e0749, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ecba71f7-525f-4818-b686-c9ff63ca561f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 925e71e3-e0b8-4973-8ff8-5438a755ef84, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 282e3323-14c4-4d62-8807-8693fcbf40ea, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 3de24d1d-6129-402e-99c6-cbe1a3727280, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 7f367ce2-eb70-48d4-b9f5-2554c178c7d8, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: cf8e7366-58a5-4520-b2ad-904e641779d8, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 81f36664-5b2e-47bc-8f86-8336a864a7bb, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 5bbf29d8-d45a-494e-b486-773cde91df47, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 6c91a708-fa2b-487f-8528-4d94d9438931, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 85a1d6ba-7262-4d27-a9fc-1d1499e79005, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 056f8fb6-5500-4b9e-92d0-2be6e076a7d3, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 5fb374a2-2d09-404e-b2d8-6f14e3884eab, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:31.922317485Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 720f4076-e581-438f-9cb1-67dafb6cb05c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:31.923956445Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 720f4076-e581-438f-9cb1-67dafb6cb05c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:31.923980256Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:31.95436417Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:31.954442872Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:31.985226456Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:31.985309528Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c66b1e8-6f9c-4af5-9706-40f991312a36, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:32.012553126Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3c66b1e8-6f9c-4af5-9706-40f991312a36, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:32.012625507Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7290f55a-1449-4cf7-88e8-036470e643e8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:32.043003501Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7290f55a-1449-4cf7-88e8-036470e643e8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:32.043072003Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 669c55e0-1c27-4089-9470-6ce2af215227, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:32.067485051Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 669c55e0-1c27-4089-9470-6ce2af215227, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:32.067548643Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 44640223-615f-49ba-b60a-a001dba9c022, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:32.096659126Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 44640223-615f-49ba-b60a-a001dba9c022, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:32.096730228Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcf0a56e-6585-45f9-8061-123749957160, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:32.125184905Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bcf0a56e-6585-45f9-8061-123749957160, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:32.125247686Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a603215-e206-4aa0-8930-250f278a0d89, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:32.154164955Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0a603215-e206-4aa0-8930-250f278a0d89, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:32.154246847Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f949516-3930-421a-bae2-b58ff4ea0dc8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:32.182502249Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3f949516-3930-421a-bae2-b58ff4ea0dc8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:32.182565991Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 74802856-0b1c-433c-9e82-d4aedb5e0749, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:32.214093323Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 74802856-0b1c-433c-9e82-d4aedb5e0749, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:32.214167474Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3de24d1d-6129-402e-99c6-cbe1a3727280, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:32.242446787Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3de24d1d-6129-402e-99c6-cbe1a3727280, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:32.242502809Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f367ce2-eb70-48d4-b9f5-2554c178c7d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:32.270074674Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7f367ce2-eb70-48d4-b9f5-2554c178c7d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:32.270123035Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cf8e7366-58a5-4520-b2ad-904e641779d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:32.291850637Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cf8e7366-58a5-4520-b2ad-904e641779d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:32.291923169Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 81f36664-5b2e-47bc-8f86-8336a864a7bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:32.312836312Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 81f36664-5b2e-47bc-8f86-8336a864a7bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:32.312891153Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5bbf29d8-d45a-494e-b486-773cde91df47, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:32.332897253Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5bbf29d8-d45a-494e-b486-773cde91df47, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:32.332946624Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c91a708-fa2b-487f-8528-4d94d9438931, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:32.352929534Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6c91a708-fa2b-487f-8528-4d94d9438931, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:32.352984215Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85a1d6ba-7262-4d27-a9fc-1d1499e79005, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:32.370714229Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 85a1d6ba-7262-4d27-a9fc-1d1499e79005, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:32.370771041Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 056f8fb6-5500-4b9e-92d0-2be6e076a7d3, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:32.390500394Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 056f8fb6-5500-4b9e-92d0-2be6e076a7d3, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:32.390546655Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5fb374a2-2d09-404e-b2d8-6f14e3884eab, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:32.410423732Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5fb374a2-2d09-404e-b2d8-6f14e3884eab, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:32.410474163Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:32.430435163Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:32.430488064Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:32.430702289Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:32.430842313Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:32.523977084Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:32.524026335Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:32.524033045Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:32.524038836Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:32.524043036Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:32.524048336Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:32.524052666Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:32.524057106Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:32.524063296Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:32.524067506Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:32.524072826Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:32.524077166Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:32.524082477Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:32.524086857Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:32.524106417Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:32.524112037Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:32.524116237Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:32.524120638Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:32.524124818Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:32.524278731Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"78cffa24-10fd-420f-9963-90ae5157b1be","remote_addr":"[fd00:1122:3344:101::1]:42557","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d\", TimeNotSynchronized), (\"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f\", TimeNotSynchronized), (\"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8\", TimeNotSynchronized), (\"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47\", TimeNotSynchronized), (\"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea\", TimeNotSynchronized), (\"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8\", TimeNotSynchronized), (\"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749\", TimeNotSynchronized), (\"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb\", TimeNotSynchronized), (\"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897\", TimeNotSynchronized), (\"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3\", TimeNotSynchronized), (\"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9\", TimeNotSynchronized), (\"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84\", TimeNotSynchronized), (\"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280\", TimeNotSynchronized), (\"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8\", TimeNotSynchronized), (\"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89\", TimeNotSynchronized), (\"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931\", TimeNotSynchronized), (\"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005\", TimeNotSynchronized), (\"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3\", TimeNotSynchronized), (\"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab\", TimeNotSynchronized)]","latency_us":601402,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:32.524496327Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"167.278902ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"78cffa24-10fd-420f-9963-90ae5157b1be\", \"content-length\": \"124\", \"date\": \"Fri, 01 Mar 2024 23:16:32 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"78cffa24-10fd-420f-9963-90ae5157b1be\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 720f4076-e581-438f-9cb1-67dafb6cb05c, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 1c292cfd-336a-4f75-8eb1-9f4fab725484, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bb8fa3f4-707c-4207-88cf-a0bf61d629c1, kind: Service { id: 1c292cfd-336a-4f75-8eb1-9f4fab725484 }, name: Name(\"ntp-1c292cfd-336a-4f75-8eb1-9f4fab725484\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 148, 132])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3c66b1e8-6f9c-4af5-9706-40f991312a36, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 7290f55a-1449-4cf7-88e8-036470e643e8, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 669c55e0-1c27-4089-9470-6ce2af215227, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 44640223-615f-49ba-b60a-a001dba9c022, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: bcf0a56e-6585-45f9-8061-123749957160, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 0a603215-e206-4aa0-8930-250f278a0d89, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 3af2d634-ca96-4144-9913-082ef495b783, kind: Service { id: 0a603215-e206-4aa0-8930-250f278a0d89 }, name: Name(\"external-dns-0a603215-e206-4aa0-8930-250f278a0d89\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 224, 81])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 3f949516-3930-421a-bae2-b58ff4ea0dc8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: d086f287-39ab-42aa-b1b5-4d91b0388531, kind: Service { id: 3f949516-3930-421a-bae2-b58ff4ea0dc8 }, name: Name(\"external-dns-3f949516-3930-421a-bae2-b58ff4ea0dc8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 146])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 4ea96463-fca7-4b4a-8a70-4e88cc368897, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: c6ce2abe-65af-4c28-bdee-3dca5bb68c72, kind: Service { id: 4ea96463-fca7-4b4a-8a70-4e88cc368897 }, name: Name(\"nexus-4ea96463-fca7-4b4a-8a70-4e88cc368897\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 933c0d3a-d49d-431b-8f65-51222f67e8f3, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 06ae0dac-13b1-4ef0-995e-b56e4a7d3138, kind: Service { id: 933c0d3a-d49d-431b-8f65-51222f67e8f3 }, name: Name(\"nexus-933c0d3a-d49d-431b-8f65-51222f67e8f3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 46])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 264d5dd4-3eb2-4adb-b639-f0a26ab5534d, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 9eeb19bd-897d-4d56-ae7e-bd0911ac9557, kind: Service { id: 264d5dd4-3eb2-4adb-b639-f0a26ab5534d }, name: Name(\"nexus-264d5dd4-3eb2-4adb-b639-f0a26ab5534d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 168, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: a87cb16a-062c-4865-a37d-8e0d7e2bd2b9, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 74802856-0b1c-433c-9e82-d4aedb5e0749, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ecba71f7-525f-4818-b686-c9ff63ca561f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 925e71e3-e0b8-4973-8ff8-5438a755ef84, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 282e3323-14c4-4d62-8807-8693fcbf40ea, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 3de24d1d-6129-402e-99c6-cbe1a3727280, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 7f367ce2-eb70-48d4-b9f5-2554c178c7d8, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: cf8e7366-58a5-4520-b2ad-904e641779d8, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 81f36664-5b2e-47bc-8f86-8336a864a7bb, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 5bbf29d8-d45a-494e-b486-773cde91df47, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 6c91a708-fa2b-487f-8528-4d94d9438931, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 85a1d6ba-7262-4d27-a9fc-1d1499e79005, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 056f8fb6-5500-4b9e-92d0-2be6e076a7d3, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 5fb374a2-2d09-404e-b2d8-6f14e3884eab, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:32.692691566Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 720f4076-e581-438f-9cb1-67dafb6cb05c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:32.693342742Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 720f4076-e581-438f-9cb1-67dafb6cb05c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:32.693366483Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:32.7152748Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:32.715319641Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:32.736568371Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:32.736608672Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c66b1e8-6f9c-4af5-9706-40f991312a36, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:32.756755565Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3c66b1e8-6f9c-4af5-9706-40f991312a36, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:32.756794777Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7290f55a-1449-4cf7-88e8-036470e643e8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:32.777127495Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7290f55a-1449-4cf7-88e8-036470e643e8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:32.777163816Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 669c55e0-1c27-4089-9470-6ce2af215227, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:32.798799586Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 669c55e0-1c27-4089-9470-6ce2af215227, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:32.798852967Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 44640223-615f-49ba-b60a-a001dba9c022, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:32.820745813Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 44640223-615f-49ba-b60a-a001dba9c022, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:32.820787004Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcf0a56e-6585-45f9-8061-123749957160, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:32.841075141Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bcf0a56e-6585-45f9-8061-123749957160, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:32.841120872Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a603215-e206-4aa0-8930-250f278a0d89, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:32.8606023Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0a603215-e206-4aa0-8930-250f278a0d89, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:32.860643421Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f949516-3930-421a-bae2-b58ff4ea0dc8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:32.880404705Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3f949516-3930-421a-bae2-b58ff4ea0dc8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:32.880452926Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 74802856-0b1c-433c-9e82-d4aedb5e0749, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:32.900141788Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 74802856-0b1c-433c-9e82-d4aedb5e0749, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:32.900176039Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3de24d1d-6129-402e-99c6-cbe1a3727280, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:32.920128308Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3de24d1d-6129-402e-99c6-cbe1a3727280, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:32.920170219Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f367ce2-eb70-48d4-b9f5-2554c178c7d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:32.941257665Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7f367ce2-eb70-48d4-b9f5-2554c178c7d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:32.941301976Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cf8e7366-58a5-4520-b2ad-904e641779d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:32.962561327Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cf8e7366-58a5-4520-b2ad-904e641779d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:32.962606658Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 81f36664-5b2e-47bc-8f86-8336a864a7bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:32.984631918Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 81f36664-5b2e-47bc-8f86-8336a864a7bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:32.984677139Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5bbf29d8-d45a-494e-b486-773cde91df47, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:33.004515635Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5bbf29d8-d45a-494e-b486-773cde91df47, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:33.004558506Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c91a708-fa2b-487f-8528-4d94d9438931, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:33.02718494Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6c91a708-fa2b-487f-8528-4d94d9438931, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:33.027237331Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85a1d6ba-7262-4d27-a9fc-1d1499e79005, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:33.047964929Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 85a1d6ba-7262-4d27-a9fc-1d1499e79005, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:33.04800319Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 056f8fb6-5500-4b9e-92d0-2be6e076a7d3, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:33.069340433Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 056f8fb6-5500-4b9e-92d0-2be6e076a7d3, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:33.069378604Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5fb374a2-2d09-404e-b2d8-6f14e3884eab, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:33.090329367Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5fb374a2-2d09-404e-b2d8-6f14e3884eab, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:33.090368588Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:33.110276436Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:33.110331257Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:33.110513231Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:33.110652835Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:33.210118822Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:33.210241405Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:33.210251465Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:33.210259655Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:33.210272385Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:33.210279026Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:33.210298886Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:33.210304726Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:33.210311926Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:33.210318296Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:33.210325657Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:33.210331457Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:33.210351437Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:33.210356817Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:33.210365838Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:33.210371798Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:33.210384378Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:33.210390388Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:33.210396248Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:33.21045768Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"d9069724-58e9-4097-be49-8c08e4056ae2","remote_addr":"[fd00:1122:3344:101::1]:42557","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8\", TimeNotSynchronized), (\"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea\", TimeNotSynchronized), (\"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d\", TimeNotSynchronized), (\"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84\", TimeNotSynchronized), (\"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8\", TimeNotSynchronized), (\"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3\", TimeNotSynchronized), (\"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931\", TimeNotSynchronized), (\"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005\", TimeNotSynchronized), (\"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749\", TimeNotSynchronized), (\"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f\", TimeNotSynchronized), (\"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897\", TimeNotSynchronized), (\"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280\", TimeNotSynchronized), (\"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8\", TimeNotSynchronized), (\"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb\", TimeNotSynchronized), (\"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3\", TimeNotSynchronized), (\"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab\", TimeNotSynchronized), (\"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47\", TimeNotSynchronized), (\"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9\", TimeNotSynchronized), (\"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89\", TimeNotSynchronized)]","latency_us":517300,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:33.210697146Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"146.674132ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d9069724-58e9-4097-be49-8c08e4056ae2\", \"content-length\": \"124\", \"date\": \"Fri, 01 Mar 2024 23:16:33 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d9069724-58e9-4097-be49-8c08e4056ae2\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 720f4076-e581-438f-9cb1-67dafb6cb05c, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 1c292cfd-336a-4f75-8eb1-9f4fab725484, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bb8fa3f4-707c-4207-88cf-a0bf61d629c1, kind: Service { id: 1c292cfd-336a-4f75-8eb1-9f4fab725484 }, name: Name(\"ntp-1c292cfd-336a-4f75-8eb1-9f4fab725484\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 148, 132])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3c66b1e8-6f9c-4af5-9706-40f991312a36, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 7290f55a-1449-4cf7-88e8-036470e643e8, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 669c55e0-1c27-4089-9470-6ce2af215227, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 44640223-615f-49ba-b60a-a001dba9c022, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: bcf0a56e-6585-45f9-8061-123749957160, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 0a603215-e206-4aa0-8930-250f278a0d89, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 3af2d634-ca96-4144-9913-082ef495b783, kind: Service { id: 0a603215-e206-4aa0-8930-250f278a0d89 }, name: Name(\"external-dns-0a603215-e206-4aa0-8930-250f278a0d89\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 224, 81])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 3f949516-3930-421a-bae2-b58ff4ea0dc8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: d086f287-39ab-42aa-b1b5-4d91b0388531, kind: Service { id: 3f949516-3930-421a-bae2-b58ff4ea0dc8 }, name: Name(\"external-dns-3f949516-3930-421a-bae2-b58ff4ea0dc8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 146])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 4ea96463-fca7-4b4a-8a70-4e88cc368897, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: c6ce2abe-65af-4c28-bdee-3dca5bb68c72, kind: Service { id: 4ea96463-fca7-4b4a-8a70-4e88cc368897 }, name: Name(\"nexus-4ea96463-fca7-4b4a-8a70-4e88cc368897\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 933c0d3a-d49d-431b-8f65-51222f67e8f3, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 06ae0dac-13b1-4ef0-995e-b56e4a7d3138, kind: Service { id: 933c0d3a-d49d-431b-8f65-51222f67e8f3 }, name: Name(\"nexus-933c0d3a-d49d-431b-8f65-51222f67e8f3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 46])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 264d5dd4-3eb2-4adb-b639-f0a26ab5534d, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 9eeb19bd-897d-4d56-ae7e-bd0911ac9557, kind: Service { id: 264d5dd4-3eb2-4adb-b639-f0a26ab5534d }, name: Name(\"nexus-264d5dd4-3eb2-4adb-b639-f0a26ab5534d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 168, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: a87cb16a-062c-4865-a37d-8e0d7e2bd2b9, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 74802856-0b1c-433c-9e82-d4aedb5e0749, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ecba71f7-525f-4818-b686-c9ff63ca561f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 925e71e3-e0b8-4973-8ff8-5438a755ef84, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 282e3323-14c4-4d62-8807-8693fcbf40ea, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 3de24d1d-6129-402e-99c6-cbe1a3727280, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 7f367ce2-eb70-48d4-b9f5-2554c178c7d8, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: cf8e7366-58a5-4520-b2ad-904e641779d8, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 81f36664-5b2e-47bc-8f86-8336a864a7bb, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 5bbf29d8-d45a-494e-b486-773cde91df47, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 6c91a708-fa2b-487f-8528-4d94d9438931, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 85a1d6ba-7262-4d27-a9fc-1d1499e79005, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 056f8fb6-5500-4b9e-92d0-2be6e076a7d3, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 5fb374a2-2d09-404e-b2d8-6f14e3884eab, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:33.357891382Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 720f4076-e581-438f-9cb1-67dafb6cb05c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:33.358531527Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 720f4076-e581-438f-9cb1-67dafb6cb05c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:33.358554118Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:33.380432484Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:33.380495795Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:33.40069033Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:33.400745151Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c66b1e8-6f9c-4af5-9706-40f991312a36, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:33.420826693Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3c66b1e8-6f9c-4af5-9706-40f991312a36, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:33.420885115Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7290f55a-1449-4cf7-88e8-036470e643e8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:33.440417023Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7290f55a-1449-4cf7-88e8-036470e643e8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:33.440486105Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 669c55e0-1c27-4089-9470-6ce2af215227, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:33.461634433Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 669c55e0-1c27-4089-9470-6ce2af215227, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:33.461685254Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 44640223-615f-49ba-b60a-a001dba9c022, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:33.481800017Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 44640223-615f-49ba-b60a-a001dba9c022, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:33.481855558Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcf0a56e-6585-45f9-8061-123749957160, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:33.502685758Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bcf0a56e-6585-45f9-8061-123749957160, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:33.50274557Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a603215-e206-4aa0-8930-250f278a0d89, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:33.521344536Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0a603215-e206-4aa0-8930-250f278a0d89, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:33.521411987Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f949516-3930-421a-bae2-b58ff4ea0dc8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:33.540756231Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3f949516-3930-421a-bae2-b58ff4ea0dc8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:33.540811023Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 74802856-0b1c-433c-9e82-d4aedb5e0749, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:33.564034851Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 74802856-0b1c-433c-9e82-d4aedb5e0749, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:33.564097933Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3de24d1d-6129-402e-99c6-cbe1a3727280, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:33.584320828Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3de24d1d-6129-402e-99c6-cbe1a3727280, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:33.58438059Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f367ce2-eb70-48d4-b9f5-2554c178c7d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:33.606356118Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7f367ce2-eb70-48d4-b9f5-2554c178c7d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:33.606407519Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cf8e7366-58a5-4520-b2ad-904e641779d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:33.627771163Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cf8e7366-58a5-4520-b2ad-904e641779d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:33.627826304Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 81f36664-5b2e-47bc-8f86-8336a864a7bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:33.645875016Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 81f36664-5b2e-47bc-8f86-8336a864a7bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:33.645932887Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5bbf29d8-d45a-494e-b486-773cde91df47, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:33.663649221Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5bbf29d8-d45a-494e-b486-773cde91df47, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:33.663707853Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c91a708-fa2b-487f-8528-4d94d9438931, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:33.685740523Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6c91a708-fa2b-487f-8528-4d94d9438931, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:33.685802724Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85a1d6ba-7262-4d27-a9fc-1d1499e79005, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:33.70646315Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 85a1d6ba-7262-4d27-a9fc-1d1499e79005, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:33.706517782Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 056f8fb6-5500-4b9e-92d0-2be6e076a7d3, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:33.727169028Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 056f8fb6-5500-4b9e-92d0-2be6e076a7d3, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:33.727236979Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5fb374a2-2d09-404e-b2d8-6f14e3884eab, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:33.747036464Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5fb374a2-2d09-404e-b2d8-6f14e3884eab, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:33.747101206Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:33.767899935Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:33.767963407Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:33.768156222Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:33.768284245Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:33.866536742Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:33.866594673Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:33.866603373Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:33.866608313Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:33.866614273Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:33.866623504Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:33.866629374Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:33.866633844Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:33.866638454Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:33.866659005Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:33.866663855Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:33.866672885Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:33.866678545Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:33.866682895Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:33.866688555Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:33.866693155Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:33.866701916Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:33.866707576Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:33.866716606Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:33.866820029Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"c4fccd11-efd6-4512-812c-26a1424fa29a","remote_addr":"[fd00:1122:3344:101::1]:42557","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8\", TimeNotSynchronized), (\"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84\", TimeNotSynchronized), (\"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9\", TimeNotSynchronized), (\"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea\", TimeNotSynchronized), (\"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897\", TimeNotSynchronized), (\"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8\", TimeNotSynchronized), (\"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8\", TimeNotSynchronized), (\"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3\", TimeNotSynchronized), (\"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f\", TimeNotSynchronized), (\"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749\", TimeNotSynchronized), (\"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47\", TimeNotSynchronized), (\"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931\", TimeNotSynchronized), (\"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005\", TimeNotSynchronized), (\"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3\", TimeNotSynchronized), (\"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89\", TimeNotSynchronized), (\"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb\", TimeNotSynchronized), (\"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab\", TimeNotSynchronized), (\"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280\", TimeNotSynchronized), (\"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d\", TimeNotSynchronized)]","latency_us":508500,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:33.867043564Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"189.695338ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c4fccd11-efd6-4512-812c-26a1424fa29a\", \"content-length\": \"124\", \"date\": \"Fri, 01 Mar 2024 23:16:33 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c4fccd11-efd6-4512-812c-26a1424fa29a\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 720f4076-e581-438f-9cb1-67dafb6cb05c, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 1c292cfd-336a-4f75-8eb1-9f4fab725484, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bb8fa3f4-707c-4207-88cf-a0bf61d629c1, kind: Service { id: 1c292cfd-336a-4f75-8eb1-9f4fab725484 }, name: Name(\"ntp-1c292cfd-336a-4f75-8eb1-9f4fab725484\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 148, 132])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3c66b1e8-6f9c-4af5-9706-40f991312a36, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 7290f55a-1449-4cf7-88e8-036470e643e8, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 669c55e0-1c27-4089-9470-6ce2af215227, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 44640223-615f-49ba-b60a-a001dba9c022, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: bcf0a56e-6585-45f9-8061-123749957160, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 0a603215-e206-4aa0-8930-250f278a0d89, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 3af2d634-ca96-4144-9913-082ef495b783, kind: Service { id: 0a603215-e206-4aa0-8930-250f278a0d89 }, name: Name(\"external-dns-0a603215-e206-4aa0-8930-250f278a0d89\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 224, 81])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 3f949516-3930-421a-bae2-b58ff4ea0dc8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: d086f287-39ab-42aa-b1b5-4d91b0388531, kind: Service { id: 3f949516-3930-421a-bae2-b58ff4ea0dc8 }, name: Name(\"external-dns-3f949516-3930-421a-bae2-b58ff4ea0dc8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 146])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 4ea96463-fca7-4b4a-8a70-4e88cc368897, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: c6ce2abe-65af-4c28-bdee-3dca5bb68c72, kind: Service { id: 4ea96463-fca7-4b4a-8a70-4e88cc368897 }, name: Name(\"nexus-4ea96463-fca7-4b4a-8a70-4e88cc368897\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 933c0d3a-d49d-431b-8f65-51222f67e8f3, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 06ae0dac-13b1-4ef0-995e-b56e4a7d3138, kind: Service { id: 933c0d3a-d49d-431b-8f65-51222f67e8f3 }, name: Name(\"nexus-933c0d3a-d49d-431b-8f65-51222f67e8f3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 46])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 264d5dd4-3eb2-4adb-b639-f0a26ab5534d, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 9eeb19bd-897d-4d56-ae7e-bd0911ac9557, kind: Service { id: 264d5dd4-3eb2-4adb-b639-f0a26ab5534d }, name: Name(\"nexus-264d5dd4-3eb2-4adb-b639-f0a26ab5534d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 168, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: a87cb16a-062c-4865-a37d-8e0d7e2bd2b9, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 74802856-0b1c-433c-9e82-d4aedb5e0749, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ecba71f7-525f-4818-b686-c9ff63ca561f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 925e71e3-e0b8-4973-8ff8-5438a755ef84, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 282e3323-14c4-4d62-8807-8693fcbf40ea, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 3de24d1d-6129-402e-99c6-cbe1a3727280, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 7f367ce2-eb70-48d4-b9f5-2554c178c7d8, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: cf8e7366-58a5-4520-b2ad-904e641779d8, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 81f36664-5b2e-47bc-8f86-8336a864a7bb, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 5bbf29d8-d45a-494e-b486-773cde91df47, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 6c91a708-fa2b-487f-8528-4d94d9438931, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 85a1d6ba-7262-4d27-a9fc-1d1499e79005, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 056f8fb6-5500-4b9e-92d0-2be6e076a7d3, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 5fb374a2-2d09-404e-b2d8-6f14e3884eab, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.058491664Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 720f4076-e581-438f-9cb1-67dafb6cb05c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.05913193Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 720f4076-e581-438f-9cb1-67dafb6cb05c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.059171571Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.082482852Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.082535873Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.103442475Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.103485726Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c66b1e8-6f9c-4af5-9706-40f991312a36, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.125599058Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3c66b1e8-6f9c-4af5-9706-40f991312a36, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.125647199Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7290f55a-1449-4cf7-88e8-036470e643e8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.145826523Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7290f55a-1449-4cf7-88e8-036470e643e8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.145871094Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 669c55e0-1c27-4089-9470-6ce2af215227, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.16527768Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 669c55e0-1c27-4089-9470-6ce2af215227, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.165322361Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 44640223-615f-49ba-b60a-a001dba9c022, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.184518071Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 44640223-615f-49ba-b60a-a001dba9c022, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.184564802Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcf0a56e-6585-45f9-8061-123749957160, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.20405501Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bcf0a56e-6585-45f9-8061-123749957160, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.204108211Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a603215-e206-4aa0-8930-250f278a0d89, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.225190228Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0a603215-e206-4aa0-8930-250f278a0d89, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.225236399Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f949516-3930-421a-bae2-b58ff4ea0dc8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.243837104Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3f949516-3930-421a-bae2-b58ff4ea0dc8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.243881035Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 74802856-0b1c-433c-9e82-d4aedb5e0749, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.263176858Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 74802856-0b1c-433c-9e82-d4aedb5e0749, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.263216139Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3de24d1d-6129-402e-99c6-cbe1a3727280, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.283553637Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3de24d1d-6129-402e-99c6-cbe1a3727280, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.283625739Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f367ce2-eb70-48d4-b9f5-2554c178c7d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.305420333Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7f367ce2-eb70-48d4-b9f5-2554c178c7d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.305481644Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cf8e7366-58a5-4520-b2ad-904e641779d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.325371752Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cf8e7366-58a5-4520-b2ad-904e641779d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.325427223Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 81f36664-5b2e-47bc-8f86-8336a864a7bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.34410191Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 81f36664-5b2e-47bc-8f86-8336a864a7bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.344159592Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5bbf29d8-d45a-494e-b486-773cde91df47, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.363864705Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5bbf29d8-d45a-494e-b486-773cde91df47, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.363932516Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c91a708-fa2b-487f-8528-4d94d9438931, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.382582313Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6c91a708-fa2b-487f-8528-4d94d9438931, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.382636305Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.393433359Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:34525"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.396937705Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"987e68ad-ce4e-42e0-a66d-85d2df65ba55","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:34525","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":3339,"response_code":"200"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85a1d6ba-7262-4d27-a9fc-1d1499e79005, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.404573732Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 85a1d6ba-7262-4d27-a9fc-1d1499e79005, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.404596132Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 056f8fb6-5500-4b9e-92d0-2be6e076a7d3, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.425700989Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 056f8fb6-5500-4b9e-92d0-2be6e076a7d3, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.425767241Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5fb374a2-2d09-404e-b2d8-6f14e3884eab, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.445626908Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5fb374a2-2d09-404e-b2d8-6f14e3884eab, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.445684529Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.464849539Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.46492094Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.465109585Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.465248158Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:34.588507207Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:34.588567158Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:34.588574509Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:34.588598679Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:34.588603659Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:34.58861356Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:34.58861831Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:34.58862592Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:34.58863127Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:34.58863574Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:34.58864104Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:34.58864553Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:34.588654811Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:34.588659451Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:34.588667761Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:34.588673551Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:34.588678971Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:34.588686501Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:34.588696762Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.588769974Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"c6d002ae-58c0-4e14-81e6-fd3860d99abf","remote_addr":"[fd00:1122:3344:101::1]:42557","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8\", TimeNotSynchronized), (\"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9\", TimeNotSynchronized), (\"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8\", TimeNotSynchronized), (\"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d\", TimeNotSynchronized), (\"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f\", TimeNotSynchronized), (\"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb\", TimeNotSynchronized), (\"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea\", TimeNotSynchronized), (\"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749\", TimeNotSynchronized), (\"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897\", TimeNotSynchronized), (\"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47\", TimeNotSynchronized), (\"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931\", TimeNotSynchronized), (\"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8\", TimeNotSynchronized), (\"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005\", TimeNotSynchronized), (\"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3\", TimeNotSynchronized), (\"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab\", TimeNotSynchronized), (\"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280\", TimeNotSynchronized), (\"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84\", TimeNotSynchronized), (\"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89\", TimeNotSynchronized), (\"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3\", TimeNotSynchronized)]","latency_us":529847,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:34.588981919Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"125.155163ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c6d002ae-58c0-4e14-81e6-fd3860d99abf\", \"content-length\": \"124\", \"date\": \"Fri, 01 Mar 2024 23:16:33 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c6d002ae-58c0-4e14-81e6-fd3860d99abf\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 720f4076-e581-438f-9cb1-67dafb6cb05c, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 1c292cfd-336a-4f75-8eb1-9f4fab725484, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bb8fa3f4-707c-4207-88cf-a0bf61d629c1, kind: Service { id: 1c292cfd-336a-4f75-8eb1-9f4fab725484 }, name: Name(\"ntp-1c292cfd-336a-4f75-8eb1-9f4fab725484\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 148, 132])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3c66b1e8-6f9c-4af5-9706-40f991312a36, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 7290f55a-1449-4cf7-88e8-036470e643e8, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 669c55e0-1c27-4089-9470-6ce2af215227, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 44640223-615f-49ba-b60a-a001dba9c022, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: bcf0a56e-6585-45f9-8061-123749957160, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 0a603215-e206-4aa0-8930-250f278a0d89, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 3af2d634-ca96-4144-9913-082ef495b783, kind: Service { id: 0a603215-e206-4aa0-8930-250f278a0d89 }, name: Name(\"external-dns-0a603215-e206-4aa0-8930-250f278a0d89\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 224, 81])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 3f949516-3930-421a-bae2-b58ff4ea0dc8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: d086f287-39ab-42aa-b1b5-4d91b0388531, kind: Service { id: 3f949516-3930-421a-bae2-b58ff4ea0dc8 }, name: Name(\"external-dns-3f949516-3930-421a-bae2-b58ff4ea0dc8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 146])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 4ea96463-fca7-4b4a-8a70-4e88cc368897, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: c6ce2abe-65af-4c28-bdee-3dca5bb68c72, kind: Service { id: 4ea96463-fca7-4b4a-8a70-4e88cc368897 }, name: Name(\"nexus-4ea96463-fca7-4b4a-8a70-4e88cc368897\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 933c0d3a-d49d-431b-8f65-51222f67e8f3, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 06ae0dac-13b1-4ef0-995e-b56e4a7d3138, kind: Service { id: 933c0d3a-d49d-431b-8f65-51222f67e8f3 }, name: Name(\"nexus-933c0d3a-d49d-431b-8f65-51222f67e8f3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 46])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 264d5dd4-3eb2-4adb-b639-f0a26ab5534d, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 9eeb19bd-897d-4d56-ae7e-bd0911ac9557, kind: Service { id: 264d5dd4-3eb2-4adb-b639-f0a26ab5534d }, name: Name(\"nexus-264d5dd4-3eb2-4adb-b639-f0a26ab5534d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 168, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: a87cb16a-062c-4865-a37d-8e0d7e2bd2b9, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 74802856-0b1c-433c-9e82-d4aedb5e0749, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ecba71f7-525f-4818-b686-c9ff63ca561f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 925e71e3-e0b8-4973-8ff8-5438a755ef84, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 282e3323-14c4-4d62-8807-8693fcbf40ea, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 3de24d1d-6129-402e-99c6-cbe1a3727280, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 7f367ce2-eb70-48d4-b9f5-2554c178c7d8, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: cf8e7366-58a5-4520-b2ad-904e641779d8, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 81f36664-5b2e-47bc-8f86-8336a864a7bb, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 5bbf29d8-d45a-494e-b486-773cde91df47, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 6c91a708-fa2b-487f-8528-4d94d9438931, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 85a1d6ba-7262-4d27-a9fc-1d1499e79005, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 056f8fb6-5500-4b9e-92d0-2be6e076a7d3, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 5fb374a2-2d09-404e-b2d8-6f14e3884eab, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.715417876Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 720f4076-e581-438f-9cb1-67dafb6cb05c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.716032321Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 720f4076-e581-438f-9cb1-67dafb6cb05c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.716049951Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.738594844Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.738646555Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.758700046Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.758752398Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c66b1e8-6f9c-4af5-9706-40f991312a36, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.77764664Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3c66b1e8-6f9c-4af5-9706-40f991312a36, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.777686891Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7290f55a-1449-4cf7-88e8-036470e643e8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.795780574Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7290f55a-1449-4cf7-88e8-036470e643e8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.795819866Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 669c55e0-1c27-4089-9470-6ce2af215227, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.818973703Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 669c55e0-1c27-4089-9470-6ce2af215227, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.819029324Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 44640223-615f-49ba-b60a-a001dba9c022, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.83766288Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 44640223-615f-49ba-b60a-a001dba9c022, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.837724492Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcf0a56e-6585-45f9-8061-123749957160, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.856665126Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bcf0a56e-6585-45f9-8061-123749957160, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.856716487Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a603215-e206-4aa0-8930-250f278a0d89, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.874732099Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0a603215-e206-4aa0-8930-250f278a0d89, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.87478207Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f949516-3930-421a-bae2-b58ff4ea0dc8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.893191031Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3f949516-3930-421a-bae2-b58ff4ea0dc8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.893229212Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 74802856-0b1c-433c-9e82-d4aedb5e0749, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.912461933Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 74802856-0b1c-433c-9e82-d4aedb5e0749, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.912503164Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3de24d1d-6129-402e-99c6-cbe1a3727280, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.932965815Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3de24d1d-6129-402e-99c6-cbe1a3727280, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.933010296Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f367ce2-eb70-48d4-b9f5-2554c178c7d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.952127924Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7f367ce2-eb70-48d4-b9f5-2554c178c7d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.952175106Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cf8e7366-58a5-4520-b2ad-904e641779d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.970868354Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cf8e7366-58a5-4520-b2ad-904e641779d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.970917665Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 81f36664-5b2e-47bc-8f86-8336a864a7bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.989026788Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 81f36664-5b2e-47bc-8f86-8336a864a7bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:34.989069189Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5bbf29d8-d45a-494e-b486-773cde91df47, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:35.007569143Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5bbf29d8-d45a-494e-b486-773cde91df47, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:35.007618494Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c91a708-fa2b-487f-8528-4d94d9438931, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:35.027621384Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6c91a708-fa2b-487f-8528-4d94d9438931, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:35.027664565Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85a1d6ba-7262-4d27-a9fc-1d1499e79005, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:35.046620819Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 85a1d6ba-7262-4d27-a9fc-1d1499e79005, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:35.046671631Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 056f8fb6-5500-4b9e-92d0-2be6e076a7d3, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:35.066707671Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 056f8fb6-5500-4b9e-92d0-2be6e076a7d3, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:35.066746752Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5fb374a2-2d09-404e-b2d8-6f14e3884eab, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:35.085967603Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5fb374a2-2d09-404e-b2d8-6f14e3884eab, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:35.086010464Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:35.105477061Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:35.105542623Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:35.105732658Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:35.105875351Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:35.206793303Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:35.206856855Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:35.206866025Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:35.206877745Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:35.206886045Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:35.206894805Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:35.206900906Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:35.206908496Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:35.206914566Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:35.206920526Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:35.206927736Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:35.206933457Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:35.206947397Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:35.206953177Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:35.206958997Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:35.206979798Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:35.206985868Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:35.206989718Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:35.206994238Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:35.20708165Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"60f32df4-1799-4334-af31-d4118a38a8c4","remote_addr":"[fd00:1122:3344:101::1]:42557","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f\", TimeNotSynchronized), (\"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb\", TimeNotSynchronized), (\"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931\", TimeNotSynchronized), (\"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84\", TimeNotSynchronized), (\"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab\", TimeNotSynchronized), (\"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea\", TimeNotSynchronized), (\"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749\", TimeNotSynchronized), (\"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897\", TimeNotSynchronized), (\"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9\", TimeNotSynchronized), (\"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d\", TimeNotSynchronized), (\"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005\", TimeNotSynchronized), (\"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3\", TimeNotSynchronized), (\"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3\", TimeNotSynchronized), (\"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8\", TimeNotSynchronized), (\"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8\", TimeNotSynchronized), (\"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47\", TimeNotSynchronized), (\"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8\", TimeNotSynchronized), (\"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280\", TimeNotSynchronized), (\"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89\", TimeNotSynchronized)]","latency_us":491247,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:35.207347457Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"398.763593ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"60f32df4-1799-4334-af31-d4118a38a8c4\", \"content-length\": \"124\", \"date\": \"Fri, 01 Mar 2024 23:16:34 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"60f32df4-1799-4334-af31-d4118a38a8c4\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 720f4076-e581-438f-9cb1-67dafb6cb05c, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 1c292cfd-336a-4f75-8eb1-9f4fab725484, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bb8fa3f4-707c-4207-88cf-a0bf61d629c1, kind: Service { id: 1c292cfd-336a-4f75-8eb1-9f4fab725484 }, name: Name(\"ntp-1c292cfd-336a-4f75-8eb1-9f4fab725484\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 148, 132])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3c66b1e8-6f9c-4af5-9706-40f991312a36, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 7290f55a-1449-4cf7-88e8-036470e643e8, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 669c55e0-1c27-4089-9470-6ce2af215227, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 44640223-615f-49ba-b60a-a001dba9c022, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: bcf0a56e-6585-45f9-8061-123749957160, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 0a603215-e206-4aa0-8930-250f278a0d89, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 3af2d634-ca96-4144-9913-082ef495b783, kind: Service { id: 0a603215-e206-4aa0-8930-250f278a0d89 }, name: Name(\"external-dns-0a603215-e206-4aa0-8930-250f278a0d89\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 224, 81])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 3f949516-3930-421a-bae2-b58ff4ea0dc8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: d086f287-39ab-42aa-b1b5-4d91b0388531, kind: Service { id: 3f949516-3930-421a-bae2-b58ff4ea0dc8 }, name: Name(\"external-dns-3f949516-3930-421a-bae2-b58ff4ea0dc8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 146])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 4ea96463-fca7-4b4a-8a70-4e88cc368897, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: c6ce2abe-65af-4c28-bdee-3dca5bb68c72, kind: Service { id: 4ea96463-fca7-4b4a-8a70-4e88cc368897 }, name: Name(\"nexus-4ea96463-fca7-4b4a-8a70-4e88cc368897\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 933c0d3a-d49d-431b-8f65-51222f67e8f3, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 06ae0dac-13b1-4ef0-995e-b56e4a7d3138, kind: Service { id: 933c0d3a-d49d-431b-8f65-51222f67e8f3 }, name: Name(\"nexus-933c0d3a-d49d-431b-8f65-51222f67e8f3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 46])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 264d5dd4-3eb2-4adb-b639-f0a26ab5534d, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 9eeb19bd-897d-4d56-ae7e-bd0911ac9557, kind: Service { id: 264d5dd4-3eb2-4adb-b639-f0a26ab5534d }, name: Name(\"nexus-264d5dd4-3eb2-4adb-b639-f0a26ab5534d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 168, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: a87cb16a-062c-4865-a37d-8e0d7e2bd2b9, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 74802856-0b1c-433c-9e82-d4aedb5e0749, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ecba71f7-525f-4818-b686-c9ff63ca561f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 925e71e3-e0b8-4973-8ff8-5438a755ef84, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 282e3323-14c4-4d62-8807-8693fcbf40ea, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 3de24d1d-6129-402e-99c6-cbe1a3727280, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 7f367ce2-eb70-48d4-b9f5-2554c178c7d8, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: cf8e7366-58a5-4520-b2ad-904e641779d8, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 81f36664-5b2e-47bc-8f86-8336a864a7bb, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 5bbf29d8-d45a-494e-b486-773cde91df47, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 6c91a708-fa2b-487f-8528-4d94d9438931, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 85a1d6ba-7262-4d27-a9fc-1d1499e79005, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 056f8fb6-5500-4b9e-92d0-2be6e076a7d3, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 5fb374a2-2d09-404e-b2d8-6f14e3884eab, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:35.60674374Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 720f4076-e581-438f-9cb1-67dafb6cb05c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:35.607400226Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 720f4076-e581-438f-9cb1-67dafb6cb05c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:35.607416426Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:35.628099623Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:35.628166175Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:35.648041281Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:35.648092973Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c66b1e8-6f9c-4af5-9706-40f991312a36, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:35.666165935Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3c66b1e8-6f9c-4af5-9706-40f991312a36, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:35.666222667Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7290f55a-1449-4cf7-88e8-036470e643e8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:35.684574256Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7290f55a-1449-4cf7-88e8-036470e643e8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:35.684628088Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 669c55e0-1c27-4089-9470-6ce2af215227, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:35.702539337Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 669c55e0-1c27-4089-9470-6ce2af215227, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:35.702598748Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 44640223-615f-49ba-b60a-a001dba9c022, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:35.720490906Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 44640223-615f-49ba-b60a-a001dba9c022, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:35.720541378Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcf0a56e-6585-45f9-8061-123749957160, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:35.738397245Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bcf0a56e-6585-45f9-8061-123749957160, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:35.738461977Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a603215-e206-4aa0-8930-250f278a0d89, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:35.756470378Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0a603215-e206-4aa0-8930-250f278a0d89, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:35.756519269Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f949516-3930-421a-bae2-b58ff4ea0dc8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:35.77574173Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3f949516-3930-421a-bae2-b58ff4ea0dc8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:35.775797241Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 74802856-0b1c-433c-9e82-d4aedb5e0749, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:35.794985921Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 74802856-0b1c-433c-9e82-d4aedb5e0749, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:35.795053443Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3de24d1d-6129-402e-99c6-cbe1a3727280, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:35.814083179Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3de24d1d-6129-402e-99c6-cbe1a3727280, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:35.81413872Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f367ce2-eb70-48d4-b9f5-2554c178c7d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:35.833053394Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7f367ce2-eb70-48d4-b9f5-2554c178c7d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:35.833106905Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cf8e7366-58a5-4520-b2ad-904e641779d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:35.852194352Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cf8e7366-58a5-4520-b2ad-904e641779d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:35.852253564Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 81f36664-5b2e-47bc-8f86-8336a864a7bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:35.871805693Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 81f36664-5b2e-47bc-8f86-8336a864a7bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:35.871859034Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5bbf29d8-d45a-494e-b486-773cde91df47, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:35.891198328Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5bbf29d8-d45a-494e-b486-773cde91df47, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:35.891249419Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c91a708-fa2b-487f-8528-4d94d9438931, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:35.912251864Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6c91a708-fa2b-487f-8528-4d94d9438931, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:35.912315345Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85a1d6ba-7262-4d27-a9fc-1d1499e79005, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:35.931738851Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 85a1d6ba-7262-4d27-a9fc-1d1499e79005, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:35.931795702Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 056f8fb6-5500-4b9e-92d0-2be6e076a7d3, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:35.951134146Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 056f8fb6-5500-4b9e-92d0-2be6e076a7d3, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:35.951184748Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5fb374a2-2d09-404e-b2d8-6f14e3884eab, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:35.97005653Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5fb374a2-2d09-404e-b2d8-6f14e3884eab, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:35.970110401Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:35.989414644Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:35.989493676Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:35.989699791Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:35.989838284Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:36.088392219Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:36.088475331Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:36.088484371Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:36.088505992Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:36.088513522Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:36.088519622Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:36.088527432Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:36.088532122Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:36.088540022Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:36.088547062Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:36.088554763Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:36.088567003Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:36.088583733Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:36.088598264Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:36.088604084Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:36.088617344Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:36.088623174Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:36.088630135Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:36.088636195Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:36.088736187Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"3c5562eb-2bf4-4890-b514-8c0a09a1724e","remote_addr":"[fd00:1122:3344:101::1]:42557","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8\", TimeNotSynchronized), (\"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d\", TimeNotSynchronized), (\"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84\", TimeNotSynchronized), (\"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea\", TimeNotSynchronized), (\"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280\", TimeNotSynchronized), (\"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8\", TimeNotSynchronized), (\"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb\", TimeNotSynchronized), (\"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005\", TimeNotSynchronized), (\"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47\", TimeNotSynchronized), (\"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3\", TimeNotSynchronized), (\"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8\", TimeNotSynchronized), (\"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931\", TimeNotSynchronized), (\"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9\", TimeNotSynchronized), (\"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749\", TimeNotSynchronized), (\"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab\", TimeNotSynchronized), (\"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89\", TimeNotSynchronized), (\"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897\", TimeNotSynchronized), (\"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3\", TimeNotSynchronized), (\"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f\", TimeNotSynchronized)]","latency_us":481529,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:36.088994113Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"467.745682ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3c5562eb-2bf4-4890-b514-8c0a09a1724e\", \"content-length\": \"124\", \"date\": \"Fri, 01 Mar 2024 23:16:36 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3c5562eb-2bf4-4890-b514-8c0a09a1724e\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 720f4076-e581-438f-9cb1-67dafb6cb05c, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 1c292cfd-336a-4f75-8eb1-9f4fab725484, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bb8fa3f4-707c-4207-88cf-a0bf61d629c1, kind: Service { id: 1c292cfd-336a-4f75-8eb1-9f4fab725484 }, name: Name(\"ntp-1c292cfd-336a-4f75-8eb1-9f4fab725484\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 148, 132])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3c66b1e8-6f9c-4af5-9706-40f991312a36, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 7290f55a-1449-4cf7-88e8-036470e643e8, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 669c55e0-1c27-4089-9470-6ce2af215227, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 44640223-615f-49ba-b60a-a001dba9c022, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: bcf0a56e-6585-45f9-8061-123749957160, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 0a603215-e206-4aa0-8930-250f278a0d89, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 3af2d634-ca96-4144-9913-082ef495b783, kind: Service { id: 0a603215-e206-4aa0-8930-250f278a0d89 }, name: Name(\"external-dns-0a603215-e206-4aa0-8930-250f278a0d89\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 224, 81])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 3f949516-3930-421a-bae2-b58ff4ea0dc8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: d086f287-39ab-42aa-b1b5-4d91b0388531, kind: Service { id: 3f949516-3930-421a-bae2-b58ff4ea0dc8 }, name: Name(\"external-dns-3f949516-3930-421a-bae2-b58ff4ea0dc8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 146])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 4ea96463-fca7-4b4a-8a70-4e88cc368897, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: c6ce2abe-65af-4c28-bdee-3dca5bb68c72, kind: Service { id: 4ea96463-fca7-4b4a-8a70-4e88cc368897 }, name: Name(\"nexus-4ea96463-fca7-4b4a-8a70-4e88cc368897\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 933c0d3a-d49d-431b-8f65-51222f67e8f3, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 06ae0dac-13b1-4ef0-995e-b56e4a7d3138, kind: Service { id: 933c0d3a-d49d-431b-8f65-51222f67e8f3 }, name: Name(\"nexus-933c0d3a-d49d-431b-8f65-51222f67e8f3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 46])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 264d5dd4-3eb2-4adb-b639-f0a26ab5534d, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 9eeb19bd-897d-4d56-ae7e-bd0911ac9557, kind: Service { id: 264d5dd4-3eb2-4adb-b639-f0a26ab5534d }, name: Name(\"nexus-264d5dd4-3eb2-4adb-b639-f0a26ab5534d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 168, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: a87cb16a-062c-4865-a37d-8e0d7e2bd2b9, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 74802856-0b1c-433c-9e82-d4aedb5e0749, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ecba71f7-525f-4818-b686-c9ff63ca561f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 925e71e3-e0b8-4973-8ff8-5438a755ef84, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 282e3323-14c4-4d62-8807-8693fcbf40ea, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 3de24d1d-6129-402e-99c6-cbe1a3727280, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 7f367ce2-eb70-48d4-b9f5-2554c178c7d8, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: cf8e7366-58a5-4520-b2ad-904e641779d8, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 81f36664-5b2e-47bc-8f86-8336a864a7bb, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 5bbf29d8-d45a-494e-b486-773cde91df47, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 6c91a708-fa2b-487f-8528-4d94d9438931, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 85a1d6ba-7262-4d27-a9fc-1d1499e79005, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 056f8fb6-5500-4b9e-92d0-2be6e076a7d3, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 5fb374a2-2d09-404e-b2d8-6f14e3884eab, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:36.558347171Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 720f4076-e581-438f-9cb1-67dafb6cb05c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:36.559001137Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 720f4076-e581-438f-9cb1-67dafb6cb05c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:36.559028318Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:36.582480872Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:36.582547064Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:36.601583049Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:36.601637541Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c66b1e8-6f9c-4af5-9706-40f991312a36, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:36.619592501Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3c66b1e8-6f9c-4af5-9706-40f991312a36, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:36.619649152Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7290f55a-1449-4cf7-88e8-036470e643e8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:36.638304399Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7290f55a-1449-4cf7-88e8-036470e643e8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:36.63835566Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 669c55e0-1c27-4089-9470-6ce2af215227, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:36.656597797Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 669c55e0-1c27-4089-9470-6ce2af215227, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:36.656655528Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 44640223-615f-49ba-b60a-a001dba9c022, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:36.674863495Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 44640223-615f-49ba-b60a-a001dba9c022, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:36.674932146Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcf0a56e-6585-45f9-8061-123749957160, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:36.69303426Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bcf0a56e-6585-45f9-8061-123749957160, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:36.693097491Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a603215-e206-4aa0-8930-250f278a0d89, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:36.710655531Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0a603215-e206-4aa0-8930-250f278a0d89, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:36.710714593Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f949516-3930-421a-bae2-b58ff4ea0dc8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:36.728679593Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3f949516-3930-421a-bae2-b58ff4ea0dc8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:36.728736114Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 74802856-0b1c-433c-9e82-d4aedb5e0749, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:36.748355275Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 74802856-0b1c-433c-9e82-d4aedb5e0749, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:36.748407316Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3de24d1d-6129-402e-99c6-cbe1a3727280, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:36.767017162Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3de24d1d-6129-402e-99c6-cbe1a3727280, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:36.767075763Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f367ce2-eb70-48d4-b9f5-2554c178c7d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:36.78448588Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7f367ce2-eb70-48d4-b9f5-2554c178c7d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:36.784553411Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cf8e7366-58a5-4520-b2ad-904e641779d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:36.80325858Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cf8e7366-58a5-4520-b2ad-904e641779d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:36.803311761Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 81f36664-5b2e-47bc-8f86-8336a864a7bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:36.820355568Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 81f36664-5b2e-47bc-8f86-8336a864a7bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:36.82041886Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5bbf29d8-d45a-494e-b486-773cde91df47, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:36.838402781Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5bbf29d8-d45a-494e-b486-773cde91df47, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:36.838466462Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c91a708-fa2b-487f-8528-4d94d9438931, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:36.855627303Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6c91a708-fa2b-487f-8528-4d94d9438931, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:36.855675954Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85a1d6ba-7262-4d27-a9fc-1d1499e79005, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:36.873454059Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 85a1d6ba-7262-4d27-a9fc-1d1499e79005, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:36.873501541Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 056f8fb6-5500-4b9e-92d0-2be6e076a7d3, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:36.89144066Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 056f8fb6-5500-4b9e-92d0-2be6e076a7d3, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:36.891502871Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5fb374a2-2d09-404e-b2d8-6f14e3884eab, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:36.908653222Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5fb374a2-2d09-404e-b2d8-6f14e3884eab, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:36.908712043Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:36.926666433Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:36.926734305Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:36.926934899Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:36.927278968Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:37.018301468Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:37.018361829Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:37.018369819Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:37.018375579Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:37.01838303Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:37.01838983Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:37.0184022Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:37.01840791Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:37.01842Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:37.018435461Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:37.018442431Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:37.018448351Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:37.018454961Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:37.018460732Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:37.018466362Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:37.018473202Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:37.018483832Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:37.018490352Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:37.018497292Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:37.018642026Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"756584fd-06f3-458b-b1d5-4faf2f441a98","remote_addr":"[fd00:1122:3344:101::1]:42557","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3\", TimeNotSynchronized), (\"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3\", TimeNotSynchronized), (\"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea\", TimeNotSynchronized), (\"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8\", TimeNotSynchronized), (\"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47\", TimeNotSynchronized), (\"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84\", TimeNotSynchronized), (\"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab\", TimeNotSynchronized), (\"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d\", TimeNotSynchronized), (\"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f\", TimeNotSynchronized), (\"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb\", TimeNotSynchronized), (\"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8\", TimeNotSynchronized), (\"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005\", TimeNotSynchronized), (\"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749\", TimeNotSynchronized), (\"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897\", TimeNotSynchronized), (\"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89\", TimeNotSynchronized), (\"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280\", TimeNotSynchronized), (\"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8\", TimeNotSynchronized), (\"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9\", TimeNotSynchronized), (\"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931\", TimeNotSynchronized)]","latency_us":459839,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:37.018881442Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"443.988071ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"756584fd-06f3-458b-b1d5-4faf2f441a98\", \"content-length\": \"124\", \"date\": \"Fri, 01 Mar 2024 23:16:37 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"756584fd-06f3-458b-b1d5-4faf2f441a98\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 720f4076-e581-438f-9cb1-67dafb6cb05c, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 1c292cfd-336a-4f75-8eb1-9f4fab725484, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bb8fa3f4-707c-4207-88cf-a0bf61d629c1, kind: Service { id: 1c292cfd-336a-4f75-8eb1-9f4fab725484 }, name: Name(\"ntp-1c292cfd-336a-4f75-8eb1-9f4fab725484\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 148, 132])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3c66b1e8-6f9c-4af5-9706-40f991312a36, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 7290f55a-1449-4cf7-88e8-036470e643e8, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 669c55e0-1c27-4089-9470-6ce2af215227, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 44640223-615f-49ba-b60a-a001dba9c022, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: bcf0a56e-6585-45f9-8061-123749957160, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 0a603215-e206-4aa0-8930-250f278a0d89, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 3af2d634-ca96-4144-9913-082ef495b783, kind: Service { id: 0a603215-e206-4aa0-8930-250f278a0d89 }, name: Name(\"external-dns-0a603215-e206-4aa0-8930-250f278a0d89\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 224, 81])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 3f949516-3930-421a-bae2-b58ff4ea0dc8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: d086f287-39ab-42aa-b1b5-4d91b0388531, kind: Service { id: 3f949516-3930-421a-bae2-b58ff4ea0dc8 }, name: Name(\"external-dns-3f949516-3930-421a-bae2-b58ff4ea0dc8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 146])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 4ea96463-fca7-4b4a-8a70-4e88cc368897, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: c6ce2abe-65af-4c28-bdee-3dca5bb68c72, kind: Service { id: 4ea96463-fca7-4b4a-8a70-4e88cc368897 }, name: Name(\"nexus-4ea96463-fca7-4b4a-8a70-4e88cc368897\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 933c0d3a-d49d-431b-8f65-51222f67e8f3, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 06ae0dac-13b1-4ef0-995e-b56e4a7d3138, kind: Service { id: 933c0d3a-d49d-431b-8f65-51222f67e8f3 }, name: Name(\"nexus-933c0d3a-d49d-431b-8f65-51222f67e8f3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 46])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 264d5dd4-3eb2-4adb-b639-f0a26ab5534d, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 9eeb19bd-897d-4d56-ae7e-bd0911ac9557, kind: Service { id: 264d5dd4-3eb2-4adb-b639-f0a26ab5534d }, name: Name(\"nexus-264d5dd4-3eb2-4adb-b639-f0a26ab5534d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 168, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: a87cb16a-062c-4865-a37d-8e0d7e2bd2b9, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 74802856-0b1c-433c-9e82-d4aedb5e0749, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ecba71f7-525f-4818-b686-c9ff63ca561f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 925e71e3-e0b8-4973-8ff8-5438a755ef84, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 282e3323-14c4-4d62-8807-8693fcbf40ea, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 3de24d1d-6129-402e-99c6-cbe1a3727280, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 7f367ce2-eb70-48d4-b9f5-2554c178c7d8, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: cf8e7366-58a5-4520-b2ad-904e641779d8, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 81f36664-5b2e-47bc-8f86-8336a864a7bb, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 5bbf29d8-d45a-494e-b486-773cde91df47, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 6c91a708-fa2b-487f-8528-4d94d9438931, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 85a1d6ba-7262-4d27-a9fc-1d1499e79005, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 056f8fb6-5500-4b9e-92d0-2be6e076a7d3, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 5fb374a2-2d09-404e-b2d8-6f14e3884eab, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:37.463604826Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 720f4076-e581-438f-9cb1-67dafb6cb05c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:37.46417234Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 720f4076-e581-438f-9cb1-67dafb6cb05c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:37.46418169Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:37.486765893Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:37.486812955Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:37.506734493Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:37.506775704Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c66b1e8-6f9c-4af5-9706-40f991312a36, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:37.526755833Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3c66b1e8-6f9c-4af5-9706-40f991312a36, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:37.526791764Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7290f55a-1449-4cf7-88e8-036470e643e8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:37.547480361Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7290f55a-1449-4cf7-88e8-036470e643e8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:37.547536592Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 669c55e0-1c27-4089-9470-6ce2af215227, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:37.565730138Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 669c55e0-1c27-4089-9470-6ce2af215227, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:37.565769869Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 44640223-615f-49ba-b60a-a001dba9c022, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:37.58499781Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 44640223-615f-49ba-b60a-a001dba9c022, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:37.585047751Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcf0a56e-6585-45f9-8061-123749957160, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:37.603580165Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bcf0a56e-6585-45f9-8061-123749957160, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:37.603619146Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a603215-e206-4aa0-8930-250f278a0d89, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:37.621696679Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0a603215-e206-4aa0-8930-250f278a0d89, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:37.62174007Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f949516-3930-421a-bae2-b58ff4ea0dc8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:37.64092667Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3f949516-3930-421a-bae2-b58ff4ea0dc8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:37.640966621Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 74802856-0b1c-433c-9e82-d4aedb5e0749, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:37.659261469Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 74802856-0b1c-433c-9e82-d4aedb5e0749, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:37.65929761Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3de24d1d-6129-402e-99c6-cbe1a3727280, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:37.677984388Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3de24d1d-6129-402e-99c6-cbe1a3727280, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:37.678036349Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f367ce2-eb70-48d4-b9f5-2554c178c7d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:37.697567937Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7f367ce2-eb70-48d4-b9f5-2554c178c7d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:37.697611329Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cf8e7366-58a5-4520-b2ad-904e641779d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:37.717628989Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cf8e7366-58a5-4520-b2ad-904e641779d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:37.71766731Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 81f36664-5b2e-47bc-8f86-8336a864a7bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:37.738380517Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 81f36664-5b2e-47bc-8f86-8336a864a7bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:37.738420998Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5bbf29d8-d45a-494e-b486-773cde91df47, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:37.758880319Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5bbf29d8-d45a-494e-b486-773cde91df47, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:37.758934781Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c91a708-fa2b-487f-8528-4d94d9438931, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:37.778424858Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6c91a708-fa2b-487f-8528-4d94d9438931, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:37.778463779Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85a1d6ba-7262-4d27-a9fc-1d1499e79005, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:37.798325006Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 85a1d6ba-7262-4d27-a9fc-1d1499e79005, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:37.798363427Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 056f8fb6-5500-4b9e-92d0-2be6e076a7d3, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:37.818890429Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 056f8fb6-5500-4b9e-92d0-2be6e076a7d3, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:37.81892703Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5fb374a2-2d09-404e-b2d8-6f14e3884eab, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:37.839474724Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5fb374a2-2d09-404e-b2d8-6f14e3884eab, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:37.839522075Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:37.859116775Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:37.859169256Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:37.85934064Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:37.859478174Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:37.95486143Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:37.954933222Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:37.954952702Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:37.954962083Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:37.954976533Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:37.954985103Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:37.955024394Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:37.955034084Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:37.955042305Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:37.955056885Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:37.955065465Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:37.955074005Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:37.955089356Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:37.955096626Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:37.955105146Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:37.955111897Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:37.955120347Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:37.955128277Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:37.955136727Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:37.95524929Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"30ac82f6-c203-4b88-8f4c-0ec70426a607","remote_addr":"[fd00:1122:3344:101::1]:42557","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84\", TimeNotSynchronized), (\"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8\", TimeNotSynchronized), (\"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8\", TimeNotSynchronized), (\"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931\", TimeNotSynchronized), (\"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005\", TimeNotSynchronized), (\"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab\", TimeNotSynchronized), (\"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897\", TimeNotSynchronized), (\"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d\", TimeNotSynchronized), (\"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8\", TimeNotSynchronized), (\"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f\", TimeNotSynchronized), (\"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280\", TimeNotSynchronized), (\"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9\", TimeNotSynchronized), (\"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749\", TimeNotSynchronized), (\"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea\", TimeNotSynchronized), (\"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb\", TimeNotSynchronized), (\"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47\", TimeNotSynchronized), (\"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3\", TimeNotSynchronized), (\"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89\", TimeNotSynchronized), (\"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3\", TimeNotSynchronized)]","latency_us":491253,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:37.955494146Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"693.964783ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"30ac82f6-c203-4b88-8f4c-0ec70426a607\", \"content-length\": \"124\", \"date\": \"Fri, 01 Mar 2024 23:16:37 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"30ac82f6-c203-4b88-8f4c-0ec70426a607\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 720f4076-e581-438f-9cb1-67dafb6cb05c, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 1c292cfd-336a-4f75-8eb1-9f4fab725484, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bb8fa3f4-707c-4207-88cf-a0bf61d629c1, kind: Service { id: 1c292cfd-336a-4f75-8eb1-9f4fab725484 }, name: Name(\"ntp-1c292cfd-336a-4f75-8eb1-9f4fab725484\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 148, 132])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3c66b1e8-6f9c-4af5-9706-40f991312a36, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 7290f55a-1449-4cf7-88e8-036470e643e8, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 669c55e0-1c27-4089-9470-6ce2af215227, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 44640223-615f-49ba-b60a-a001dba9c022, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: bcf0a56e-6585-45f9-8061-123749957160, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 0a603215-e206-4aa0-8930-250f278a0d89, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 3af2d634-ca96-4144-9913-082ef495b783, kind: Service { id: 0a603215-e206-4aa0-8930-250f278a0d89 }, name: Name(\"external-dns-0a603215-e206-4aa0-8930-250f278a0d89\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 224, 81])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 3f949516-3930-421a-bae2-b58ff4ea0dc8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: d086f287-39ab-42aa-b1b5-4d91b0388531, kind: Service { id: 3f949516-3930-421a-bae2-b58ff4ea0dc8 }, name: Name(\"external-dns-3f949516-3930-421a-bae2-b58ff4ea0dc8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 146])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 4ea96463-fca7-4b4a-8a70-4e88cc368897, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: c6ce2abe-65af-4c28-bdee-3dca5bb68c72, kind: Service { id: 4ea96463-fca7-4b4a-8a70-4e88cc368897 }, name: Name(\"nexus-4ea96463-fca7-4b4a-8a70-4e88cc368897\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 933c0d3a-d49d-431b-8f65-51222f67e8f3, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 06ae0dac-13b1-4ef0-995e-b56e4a7d3138, kind: Service { id: 933c0d3a-d49d-431b-8f65-51222f67e8f3 }, name: Name(\"nexus-933c0d3a-d49d-431b-8f65-51222f67e8f3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 46])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 264d5dd4-3eb2-4adb-b639-f0a26ab5534d, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 9eeb19bd-897d-4d56-ae7e-bd0911ac9557, kind: Service { id: 264d5dd4-3eb2-4adb-b639-f0a26ab5534d }, name: Name(\"nexus-264d5dd4-3eb2-4adb-b639-f0a26ab5534d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 168, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: a87cb16a-062c-4865-a37d-8e0d7e2bd2b9, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 74802856-0b1c-433c-9e82-d4aedb5e0749, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ecba71f7-525f-4818-b686-c9ff63ca561f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 925e71e3-e0b8-4973-8ff8-5438a755ef84, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 282e3323-14c4-4d62-8807-8693fcbf40ea, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 3de24d1d-6129-402e-99c6-cbe1a3727280, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 7f367ce2-eb70-48d4-b9f5-2554c178c7d8, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: cf8e7366-58a5-4520-b2ad-904e641779d8, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 81f36664-5b2e-47bc-8f86-8336a864a7bb, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 5bbf29d8-d45a-494e-b486-773cde91df47, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 6c91a708-fa2b-487f-8528-4d94d9438931, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 85a1d6ba-7262-4d27-a9fc-1d1499e79005, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 056f8fb6-5500-4b9e-92d0-2be6e076a7d3, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 5fb374a2-2d09-404e-b2d8-6f14e3884eab, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:38.650961042Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 720f4076-e581-438f-9cb1-67dafb6cb05c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:38.651663369Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 720f4076-e581-438f-9cb1-67dafb6cb05c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:38.651681969Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:38.673781051Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:38.673836392Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:38.693463223Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:38.693534555Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c66b1e8-6f9c-4af5-9706-40f991312a36, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:38.713392671Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3c66b1e8-6f9c-4af5-9706-40f991312a36, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:38.713472733Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7290f55a-1449-4cf7-88e8-036470e643e8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:38.733166006Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7290f55a-1449-4cf7-88e8-036470e643e8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:38.733223887Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 669c55e0-1c27-4089-9470-6ce2af215227, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:38.751157666Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 669c55e0-1c27-4089-9470-6ce2af215227, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:38.751217968Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 44640223-615f-49ba-b60a-a001dba9c022, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:38.769998108Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 44640223-615f-49ba-b60a-a001dba9c022, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:38.77005728Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcf0a56e-6585-45f9-8061-123749957160, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:38.789438824Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bcf0a56e-6585-45f9-8061-123749957160, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:38.789496676Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a603215-e206-4aa0-8930-250f278a0d89, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:38.808180873Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0a603215-e206-4aa0-8930-250f278a0d89, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:38.808236715Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f949516-3930-421a-bae2-b58ff4ea0dc8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:38.827159868Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3f949516-3930-421a-bae2-b58ff4ea0dc8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:38.827340583Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 74802856-0b1c-433c-9e82-d4aedb5e0749, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:38.846219705Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 74802856-0b1c-433c-9e82-d4aedb5e0749, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:38.846271067Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3de24d1d-6129-402e-99c6-cbe1a3727280, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:38.864662587Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3de24d1d-6129-402e-99c6-cbe1a3727280, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:38.864713308Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f367ce2-eb70-48d4-b9f5-2554c178c7d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:38.885278862Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7f367ce2-eb70-48d4-b9f5-2554c178c7d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:38.885340233Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cf8e7366-58a5-4520-b2ad-904e641779d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:38.904194825Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cf8e7366-58a5-4520-b2ad-904e641779d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:38.904251357Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 81f36664-5b2e-47bc-8f86-8336a864a7bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:38.922264438Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 81f36664-5b2e-47bc-8f86-8336a864a7bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:38.922325939Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5bbf29d8-d45a-494e-b486-773cde91df47, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:38.939809378Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5bbf29d8-d45a-494e-b486-773cde91df47, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:38.939962442Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c91a708-fa2b-487f-8528-4d94d9438931, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:38.958573778Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6c91a708-fa2b-487f-8528-4d94d9438931, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:38.958742372Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85a1d6ba-7262-4d27-a9fc-1d1499e79005, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:38.976598789Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 85a1d6ba-7262-4d27-a9fc-1d1499e79005, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:38.97665239Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 056f8fb6-5500-4b9e-92d0-2be6e076a7d3, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:38.995080332Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 056f8fb6-5500-4b9e-92d0-2be6e076a7d3, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:38.995152603Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5fb374a2-2d09-404e-b2d8-6f14e3884eab, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:39.013628306Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5fb374a2-2d09-404e-b2d8-6f14e3884eab, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:39.016002244Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:39.031394721Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:39.031453193Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:39.031689739Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:39.031836252Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:39.125598649Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:39.12565418Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:39.125673311Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:39.125696352Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:39.125704792Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:39.125710962Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:39.125717512Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:39.125724742Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:39.125731162Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:39.125736922Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:39.125743563Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:39.125750203Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:39.125762833Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:39.125768603Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:39.125780503Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:39.125787314Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:39.125797864Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:39.125804864Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:39.125821544Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:39.125933077Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"f0cca351-d2fc-4840-9e57-615e36cf53dc","remote_addr":"[fd00:1122:3344:101::1]:42557","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749\", TimeNotSynchronized), (\"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897\", TimeNotSynchronized), (\"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8\", TimeNotSynchronized), (\"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9\", TimeNotSynchronized), (\"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb\", TimeNotSynchronized), (\"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8\", TimeNotSynchronized), (\"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab\", TimeNotSynchronized), (\"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47\", TimeNotSynchronized), (\"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea\", TimeNotSynchronized), (\"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f\", TimeNotSynchronized), (\"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3\", TimeNotSynchronized), (\"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d\", TimeNotSynchronized), (\"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8\", TimeNotSynchronized), (\"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005\", TimeNotSynchronized), (\"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89\", TimeNotSynchronized), (\"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3\", TimeNotSynchronized), (\"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931\", TimeNotSynchronized), (\"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84\", TimeNotSynchronized), (\"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280\", TimeNotSynchronized)]","latency_us":474458,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:39.126166113Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"382.365053ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f0cca351-d2fc-4840-9e57-615e36cf53dc\", \"content-length\": \"124\", \"date\": \"Fri, 01 Mar 2024 23:16:38 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f0cca351-d2fc-4840-9e57-615e36cf53dc\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 720f4076-e581-438f-9cb1-67dafb6cb05c, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 1c292cfd-336a-4f75-8eb1-9f4fab725484, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bb8fa3f4-707c-4207-88cf-a0bf61d629c1, kind: Service { id: 1c292cfd-336a-4f75-8eb1-9f4fab725484 }, name: Name(\"ntp-1c292cfd-336a-4f75-8eb1-9f4fab725484\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 148, 132])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3c66b1e8-6f9c-4af5-9706-40f991312a36, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 7290f55a-1449-4cf7-88e8-036470e643e8, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 669c55e0-1c27-4089-9470-6ce2af215227, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 44640223-615f-49ba-b60a-a001dba9c022, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: bcf0a56e-6585-45f9-8061-123749957160, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 0a603215-e206-4aa0-8930-250f278a0d89, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 3af2d634-ca96-4144-9913-082ef495b783, kind: Service { id: 0a603215-e206-4aa0-8930-250f278a0d89 }, name: Name(\"external-dns-0a603215-e206-4aa0-8930-250f278a0d89\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 224, 81])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 3f949516-3930-421a-bae2-b58ff4ea0dc8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: d086f287-39ab-42aa-b1b5-4d91b0388531, kind: Service { id: 3f949516-3930-421a-bae2-b58ff4ea0dc8 }, name: Name(\"external-dns-3f949516-3930-421a-bae2-b58ff4ea0dc8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 146])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 4ea96463-fca7-4b4a-8a70-4e88cc368897, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: c6ce2abe-65af-4c28-bdee-3dca5bb68c72, kind: Service { id: 4ea96463-fca7-4b4a-8a70-4e88cc368897 }, name: Name(\"nexus-4ea96463-fca7-4b4a-8a70-4e88cc368897\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 933c0d3a-d49d-431b-8f65-51222f67e8f3, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 06ae0dac-13b1-4ef0-995e-b56e4a7d3138, kind: Service { id: 933c0d3a-d49d-431b-8f65-51222f67e8f3 }, name: Name(\"nexus-933c0d3a-d49d-431b-8f65-51222f67e8f3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 46])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 264d5dd4-3eb2-4adb-b639-f0a26ab5534d, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 9eeb19bd-897d-4d56-ae7e-bd0911ac9557, kind: Service { id: 264d5dd4-3eb2-4adb-b639-f0a26ab5534d }, name: Name(\"nexus-264d5dd4-3eb2-4adb-b639-f0a26ab5534d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 168, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: a87cb16a-062c-4865-a37d-8e0d7e2bd2b9, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 74802856-0b1c-433c-9e82-d4aedb5e0749, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ecba71f7-525f-4818-b686-c9ff63ca561f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 925e71e3-e0b8-4973-8ff8-5438a755ef84, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 282e3323-14c4-4d62-8807-8693fcbf40ea, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 3de24d1d-6129-402e-99c6-cbe1a3727280, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 7f367ce2-eb70-48d4-b9f5-2554c178c7d8, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: cf8e7366-58a5-4520-b2ad-904e641779d8, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 81f36664-5b2e-47bc-8f86-8336a864a7bb, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 5bbf29d8-d45a-494e-b486-773cde91df47, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 6c91a708-fa2b-487f-8528-4d94d9438931, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 85a1d6ba-7262-4d27-a9fc-1d1499e79005, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 056f8fb6-5500-4b9e-92d0-2be6e076a7d3, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 5fb374a2-2d09-404e-b2d8-6f14e3884eab, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:39.510388665Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 720f4076-e581-438f-9cb1-67dafb6cb05c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:39.511534263Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 720f4076-e581-438f-9cb1-67dafb6cb05c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:39.511553984Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:39.536939336Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:39.536985857Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:39.560356269Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:39.560413011Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c66b1e8-6f9c-4af5-9706-40f991312a36, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:39.582558043Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3c66b1e8-6f9c-4af5-9706-40f991312a36, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:39.582600754Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7290f55a-1449-4cf7-88e8-036470e643e8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:39.606185161Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7290f55a-1449-4cf7-88e8-036470e643e8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:39.606231662Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 669c55e0-1c27-4089-9470-6ce2af215227, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:39.627844262Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 669c55e0-1c27-4089-9470-6ce2af215227, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:39.627888203Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 44640223-615f-49ba-b60a-a001dba9c022, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:39.647824921Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 44640223-615f-49ba-b60a-a001dba9c022, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:39.647872782Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcf0a56e-6585-45f9-8061-123749957160, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:39.66735384Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bcf0a56e-6585-45f9-8061-123749957160, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:39.667397601Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a603215-e206-4aa0-8930-250f278a0d89, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:39.685111455Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0a603215-e206-4aa0-8930-250f278a0d89, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:39.685154876Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f949516-3930-421a-bae2-b58ff4ea0dc8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:39.703475224Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3f949516-3930-421a-bae2-b58ff4ea0dc8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:39.703519895Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 74802856-0b1c-433c-9e82-d4aedb5e0749, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:39.721480445Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 74802856-0b1c-433c-9e82-d4aedb5e0749, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:39.721514986Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3de24d1d-6129-402e-99c6-cbe1a3727280, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:39.739481066Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3de24d1d-6129-402e-99c6-cbe1a3727280, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:39.739520047Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f367ce2-eb70-48d4-b9f5-2554c178c7d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:39.758877451Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7f367ce2-eb70-48d4-b9f5-2554c178c7d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:39.758913692Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cf8e7366-58a5-4520-b2ad-904e641779d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:39.775621922Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cf8e7366-58a5-4520-b2ad-904e641779d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:39.775656802Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 81f36664-5b2e-47bc-8f86-8336a864a7bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:39.793783177Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 81f36664-5b2e-47bc-8f86-8336a864a7bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:39.793822738Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5bbf29d8-d45a-494e-b486-773cde91df47, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:39.811449719Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5bbf29d8-d45a-494e-b486-773cde91df47, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:39.81148519Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c91a708-fa2b-487f-8528-4d94d9438931, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:39.829071521Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6c91a708-fa2b-487f-8528-4d94d9438931, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:39.829119582Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85a1d6ba-7262-4d27-a9fc-1d1499e79005, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:39.847057542Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 85a1d6ba-7262-4d27-a9fc-1d1499e79005, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:39.847113653Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 056f8fb6-5500-4b9e-92d0-2be6e076a7d3, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:39.865272168Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 056f8fb6-5500-4b9e-92d0-2be6e076a7d3, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:39.865308909Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5fb374a2-2d09-404e-b2d8-6f14e3884eab, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:39.884873098Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5fb374a2-2d09-404e-b2d8-6f14e3884eab, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:39.884915179Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:39.90333033Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:39.903385751Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:39.903590287Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:39.90372447Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:39.995829376Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:39.995895438Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:39.995904578Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:39.995916858Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:39.995923878Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:39.995929769Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:39.995935979Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:39.995949459Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:39.995955669Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:39.995962949Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:39.99596869Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:39.99597635Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:39.99598818Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:39.99599615Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:39.99600392Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:39.996017171Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:39.996039281Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:39.996044081Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:39.996049661Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:39.996131554Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"baa0c6e8-35a3-417e-a83f-fee9de25c1a5","remote_addr":"[fd00:1122:3344:101::1]:42557","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897\", TimeNotSynchronized), (\"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47\", TimeNotSynchronized), (\"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3\", TimeNotSynchronized), (\"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f\", TimeNotSynchronized), (\"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8\", TimeNotSynchronized), (\"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab\", TimeNotSynchronized), (\"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8\", TimeNotSynchronized), (\"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89\", TimeNotSynchronized), (\"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3\", TimeNotSynchronized), (\"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749\", TimeNotSynchronized), (\"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84\", TimeNotSynchronized), (\"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb\", TimeNotSynchronized), (\"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d\", TimeNotSynchronized), (\"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9\", TimeNotSynchronized), (\"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea\", TimeNotSynchronized), (\"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280\", TimeNotSynchronized), (\"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931\", TimeNotSynchronized), (\"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8\", TimeNotSynchronized), (\"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005\", TimeNotSynchronized)]","latency_us":485319,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:39.996366549Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"998.908985ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"baa0c6e8-35a3-417e-a83f-fee9de25c1a5\", \"content-length\": \"124\", \"date\": \"Fri, 01 Mar 2024 23:16:39 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"baa0c6e8-35a3-417e-a83f-fee9de25c1a5\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 720f4076-e581-438f-9cb1-67dafb6cb05c, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 1c292cfd-336a-4f75-8eb1-9f4fab725484, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bb8fa3f4-707c-4207-88cf-a0bf61d629c1, kind: Service { id: 1c292cfd-336a-4f75-8eb1-9f4fab725484 }, name: Name(\"ntp-1c292cfd-336a-4f75-8eb1-9f4fab725484\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 148, 132])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3c66b1e8-6f9c-4af5-9706-40f991312a36, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 7290f55a-1449-4cf7-88e8-036470e643e8, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 669c55e0-1c27-4089-9470-6ce2af215227, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 44640223-615f-49ba-b60a-a001dba9c022, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: bcf0a56e-6585-45f9-8061-123749957160, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 0a603215-e206-4aa0-8930-250f278a0d89, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 3af2d634-ca96-4144-9913-082ef495b783, kind: Service { id: 0a603215-e206-4aa0-8930-250f278a0d89 }, name: Name(\"external-dns-0a603215-e206-4aa0-8930-250f278a0d89\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 224, 81])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 3f949516-3930-421a-bae2-b58ff4ea0dc8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: d086f287-39ab-42aa-b1b5-4d91b0388531, kind: Service { id: 3f949516-3930-421a-bae2-b58ff4ea0dc8 }, name: Name(\"external-dns-3f949516-3930-421a-bae2-b58ff4ea0dc8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 146])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 4ea96463-fca7-4b4a-8a70-4e88cc368897, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: c6ce2abe-65af-4c28-bdee-3dca5bb68c72, kind: Service { id: 4ea96463-fca7-4b4a-8a70-4e88cc368897 }, name: Name(\"nexus-4ea96463-fca7-4b4a-8a70-4e88cc368897\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 933c0d3a-d49d-431b-8f65-51222f67e8f3, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 06ae0dac-13b1-4ef0-995e-b56e4a7d3138, kind: Service { id: 933c0d3a-d49d-431b-8f65-51222f67e8f3 }, name: Name(\"nexus-933c0d3a-d49d-431b-8f65-51222f67e8f3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 46])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 264d5dd4-3eb2-4adb-b639-f0a26ab5534d, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 9eeb19bd-897d-4d56-ae7e-bd0911ac9557, kind: Service { id: 264d5dd4-3eb2-4adb-b639-f0a26ab5534d }, name: Name(\"nexus-264d5dd4-3eb2-4adb-b639-f0a26ab5534d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 168, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: a87cb16a-062c-4865-a37d-8e0d7e2bd2b9, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 74802856-0b1c-433c-9e82-d4aedb5e0749, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ecba71f7-525f-4818-b686-c9ff63ca561f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 925e71e3-e0b8-4973-8ff8-5438a755ef84, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 282e3323-14c4-4d62-8807-8693fcbf40ea, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 3de24d1d-6129-402e-99c6-cbe1a3727280, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 7f367ce2-eb70-48d4-b9f5-2554c178c7d8, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: cf8e7366-58a5-4520-b2ad-904e641779d8, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 81f36664-5b2e-47bc-8f86-8336a864a7bb, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 5bbf29d8-d45a-494e-b486-773cde91df47, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 6c91a708-fa2b-487f-8528-4d94d9438931, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 85a1d6ba-7262-4d27-a9fc-1d1499e79005, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 056f8fb6-5500-4b9e-92d0-2be6e076a7d3, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 5fb374a2-2d09-404e-b2d8-6f14e3884eab, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:40.995918885Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 720f4076-e581-438f-9cb1-67dafb6cb05c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:40.996909049Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 720f4076-e581-438f-9cb1-67dafb6cb05c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:40.9969286Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:41.019825271Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:41.022265091Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:41.038000716Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:41.038065567Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c66b1e8-6f9c-4af5-9706-40f991312a36, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:41.055579677Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3c66b1e8-6f9c-4af5-9706-40f991312a36, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:41.055636918Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7290f55a-1449-4cf7-88e8-036470e643e8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:41.075568886Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7290f55a-1449-4cf7-88e8-036470e643e8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:41.075633838Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 669c55e0-1c27-4089-9470-6ce2af215227, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:41.093188848Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 669c55e0-1c27-4089-9470-6ce2af215227, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:41.093239089Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 44640223-615f-49ba-b60a-a001dba9c022, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:41.111949408Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 44640223-615f-49ba-b60a-a001dba9c022, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:41.112005809Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcf0a56e-6585-45f9-8061-123749957160, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:41.129405885Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bcf0a56e-6585-45f9-8061-123749957160, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:41.129478107Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a603215-e206-4aa0-8930-250f278a0d89, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:41.149367514Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0a603215-e206-4aa0-8930-250f278a0d89, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:41.149423475Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f949516-3930-421a-bae2-b58ff4ea0dc8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:41.167199751Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3f949516-3930-421a-bae2-b58ff4ea0dc8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:41.169364924Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 74802856-0b1c-433c-9e82-d4aedb5e0749, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:41.184644978Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 74802856-0b1c-433c-9e82-d4aedb5e0749, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:41.18471165Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3de24d1d-6129-402e-99c6-cbe1a3727280, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:41.201161923Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3de24d1d-6129-402e-99c6-cbe1a3727280, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:41.201217944Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f367ce2-eb70-48d4-b9f5-2554c178c7d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:41.217706428Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7f367ce2-eb70-48d4-b9f5-2554c178c7d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:41.21776132Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cf8e7366-58a5-4520-b2ad-904e641779d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:41.233647089Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cf8e7366-58a5-4520-b2ad-904e641779d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:41.23370663Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 81f36664-5b2e-47bc-8f86-8336a864a7bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:41.251736302Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 81f36664-5b2e-47bc-8f86-8336a864a7bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:41.251800283Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5bbf29d8-d45a-494e-b486-773cde91df47, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:41.268836611Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5bbf29d8-d45a-494e-b486-773cde91df47, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:41.268896822Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c91a708-fa2b-487f-8528-4d94d9438931, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:41.284897664Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6c91a708-fa2b-487f-8528-4d94d9438931, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:41.284962566Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85a1d6ba-7262-4d27-a9fc-1d1499e79005, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:41.30145441Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 85a1d6ba-7262-4d27-a9fc-1d1499e79005, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:41.301510871Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 056f8fb6-5500-4b9e-92d0-2be6e076a7d3, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:41.317434791Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 056f8fb6-5500-4b9e-92d0-2be6e076a7d3, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:41.317501093Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5fb374a2-2d09-404e-b2d8-6f14e3884eab, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:41.335215607Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5fb374a2-2d09-404e-b2d8-6f14e3884eab, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:41.335275348Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:41.35287695Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:41.352954731Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:41.353173547Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:41.353343201Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:41.440331082Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:41.440391963Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:41.440400274Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:41.440407194Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:41.440414254Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:41.440420934Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:41.440432794Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:41.440438384Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:41.440450375Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:41.440456995Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:41.440463585Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:41.440482665Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:41.440489586Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:41.440511306Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:41.440518057Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:41.440523746Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:41.440530757Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:41.440536337Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:41.440542977Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:41.44067402Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"34c1695b-0e9a-491a-bda9-333fdf27653b","remote_addr":"[fd00:1122:3344:101::1]:42557","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8\", TimeNotSynchronized), (\"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931\", TimeNotSynchronized), (\"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89\", TimeNotSynchronized), (\"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb\", TimeNotSynchronized), (\"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea\", TimeNotSynchronized), (\"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8\", TimeNotSynchronized), (\"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005\", TimeNotSynchronized), (\"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749\", TimeNotSynchronized), (\"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8\", TimeNotSynchronized), (\"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9\", TimeNotSynchronized), (\"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47\", TimeNotSynchronized), (\"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f\", TimeNotSynchronized), (\"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d\", TimeNotSynchronized), (\"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3\", TimeNotSynchronized), (\"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab\", TimeNotSynchronized), (\"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897\", TimeNotSynchronized), (\"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280\", TimeNotSynchronized), (\"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84\", TimeNotSynchronized), (\"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3\", TimeNotSynchronized)]","latency_us":443950,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:41.440890995Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"765.612636ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"34c1695b-0e9a-491a-bda9-333fdf27653b\", \"content-length\": \"124\", \"date\": \"Fri, 01 Mar 2024 23:16:40 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"34c1695b-0e9a-491a-bda9-333fdf27653b\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 720f4076-e581-438f-9cb1-67dafb6cb05c, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 1c292cfd-336a-4f75-8eb1-9f4fab725484, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bb8fa3f4-707c-4207-88cf-a0bf61d629c1, kind: Service { id: 1c292cfd-336a-4f75-8eb1-9f4fab725484 }, name: Name(\"ntp-1c292cfd-336a-4f75-8eb1-9f4fab725484\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 148, 132])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3c66b1e8-6f9c-4af5-9706-40f991312a36, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 7290f55a-1449-4cf7-88e8-036470e643e8, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 669c55e0-1c27-4089-9470-6ce2af215227, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 44640223-615f-49ba-b60a-a001dba9c022, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: bcf0a56e-6585-45f9-8061-123749957160, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 0a603215-e206-4aa0-8930-250f278a0d89, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 3af2d634-ca96-4144-9913-082ef495b783, kind: Service { id: 0a603215-e206-4aa0-8930-250f278a0d89 }, name: Name(\"external-dns-0a603215-e206-4aa0-8930-250f278a0d89\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 224, 81])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 3f949516-3930-421a-bae2-b58ff4ea0dc8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: d086f287-39ab-42aa-b1b5-4d91b0388531, kind: Service { id: 3f949516-3930-421a-bae2-b58ff4ea0dc8 }, name: Name(\"external-dns-3f949516-3930-421a-bae2-b58ff4ea0dc8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 146])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 4ea96463-fca7-4b4a-8a70-4e88cc368897, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: c6ce2abe-65af-4c28-bdee-3dca5bb68c72, kind: Service { id: 4ea96463-fca7-4b4a-8a70-4e88cc368897 }, name: Name(\"nexus-4ea96463-fca7-4b4a-8a70-4e88cc368897\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 933c0d3a-d49d-431b-8f65-51222f67e8f3, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 06ae0dac-13b1-4ef0-995e-b56e4a7d3138, kind: Service { id: 933c0d3a-d49d-431b-8f65-51222f67e8f3 }, name: Name(\"nexus-933c0d3a-d49d-431b-8f65-51222f67e8f3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 46])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 264d5dd4-3eb2-4adb-b639-f0a26ab5534d, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 9eeb19bd-897d-4d56-ae7e-bd0911ac9557, kind: Service { id: 264d5dd4-3eb2-4adb-b639-f0a26ab5534d }, name: Name(\"nexus-264d5dd4-3eb2-4adb-b639-f0a26ab5534d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 168, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: a87cb16a-062c-4865-a37d-8e0d7e2bd2b9, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 74802856-0b1c-433c-9e82-d4aedb5e0749, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ecba71f7-525f-4818-b686-c9ff63ca561f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 925e71e3-e0b8-4973-8ff8-5438a755ef84, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 282e3323-14c4-4d62-8807-8693fcbf40ea, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 3de24d1d-6129-402e-99c6-cbe1a3727280, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 7f367ce2-eb70-48d4-b9f5-2554c178c7d8, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: cf8e7366-58a5-4520-b2ad-904e641779d8, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 81f36664-5b2e-47bc-8f86-8336a864a7bb, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 5bbf29d8-d45a-494e-b486-773cde91df47, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 6c91a708-fa2b-487f-8528-4d94d9438931, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 85a1d6ba-7262-4d27-a9fc-1d1499e79005, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 056f8fb6-5500-4b9e-92d0-2be6e076a7d3, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 5fb374a2-2d09-404e-b2d8-6f14e3884eab, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:42.208211072Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 720f4076-e581-438f-9cb1-67dafb6cb05c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:42.20895093Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 720f4076-e581-438f-9cb1-67dafb6cb05c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:42.20896946Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:42.229730819Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:42.22979842Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:42.247492844Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:42.247535295Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c66b1e8-6f9c-4af5-9706-40f991312a36, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:42.264703155Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3c66b1e8-6f9c-4af5-9706-40f991312a36, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:42.264746076Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7290f55a-1449-4cf7-88e8-036470e643e8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:42.281835725Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7290f55a-1449-4cf7-88e8-036470e643e8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:42.281877866Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 669c55e0-1c27-4089-9470-6ce2af215227, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:42.299102008Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 669c55e0-1c27-4089-9470-6ce2af215227, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:42.299142829Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 44640223-615f-49ba-b60a-a001dba9c022, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:42.31633135Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 44640223-615f-49ba-b60a-a001dba9c022, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:42.316372151Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcf0a56e-6585-45f9-8061-123749957160, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:42.333590303Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bcf0a56e-6585-45f9-8061-123749957160, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:42.333633744Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a603215-e206-4aa0-8930-250f278a0d89, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:42.352801284Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0a603215-e206-4aa0-8930-250f278a0d89, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:42.352843515Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f949516-3930-421a-bae2-b58ff4ea0dc8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:42.370561369Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3f949516-3930-421a-bae2-b58ff4ea0dc8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:42.37060972Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 74802856-0b1c-433c-9e82-d4aedb5e0749, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:42.389702538Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 74802856-0b1c-433c-9e82-d4aedb5e0749, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:42.389745639Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3de24d1d-6129-402e-99c6-cbe1a3727280, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:42.408316304Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3de24d1d-6129-402e-99c6-cbe1a3727280, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:42.408356085Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f367ce2-eb70-48d4-b9f5-2554c178c7d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:42.426989441Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7f367ce2-eb70-48d4-b9f5-2554c178c7d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:42.427033682Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cf8e7366-58a5-4520-b2ad-904e641779d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:42.446891918Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cf8e7366-58a5-4520-b2ad-904e641779d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:42.44693704Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 81f36664-5b2e-47bc-8f86-8336a864a7bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:42.466639672Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 81f36664-5b2e-47bc-8f86-8336a864a7bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:42.466672533Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5bbf29d8-d45a-494e-b486-773cde91df47, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:42.489697947Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5bbf29d8-d45a-494e-b486-773cde91df47, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:42.489745748Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c91a708-fa2b-487f-8528-4d94d9438931, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:42.511485931Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6c91a708-fa2b-487f-8528-4d94d9438931, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:42.511531362Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85a1d6ba-7262-4d27-a9fc-1d1499e79005, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:42.531104101Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 85a1d6ba-7262-4d27-a9fc-1d1499e79005, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:42.531144013Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 056f8fb6-5500-4b9e-92d0-2be6e076a7d3, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:42.550810994Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 056f8fb6-5500-4b9e-92d0-2be6e076a7d3, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:42.550850035Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5fb374a2-2d09-404e-b2d8-6f14e3884eab, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:42.570609399Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5fb374a2-2d09-404e-b2d8-6f14e3884eab, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:42.570661961Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:42.591044059Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:42.59109019Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:42.591291585Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:42.591411038Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:42.692076804Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:42.692149196Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:42.692163616Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:42.692170966Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:42.692179947Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:42.692186407Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:42.692193947Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:42.692201397Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:42.692208327Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:42.692216598Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:42.692245378Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:42.692253988Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:42.692261379Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:42.692270309Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:42.692277589Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:42.692292829Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:42.69230135Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:42.69231568Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:42.69232436Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:42.692440213Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"8e80a662-35ae-42ab-89e2-abc60c6f21d1","remote_addr":"[fd00:1122:3344:101::1]:42557","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab\", TimeNotSynchronized), (\"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8\", TimeNotSynchronized), (\"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897\", TimeNotSynchronized), (\"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9\", TimeNotSynchronized), (\"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749\", TimeNotSynchronized), (\"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea\", TimeNotSynchronized), (\"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3\", TimeNotSynchronized), (\"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d\", TimeNotSynchronized), (\"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f\", TimeNotSynchronized), (\"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84\", TimeNotSynchronized), (\"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280\", TimeNotSynchronized), (\"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8\", TimeNotSynchronized), (\"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3\", TimeNotSynchronized), (\"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8\", TimeNotSynchronized), (\"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb\", TimeNotSynchronized), (\"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47\", TimeNotSynchronized), (\"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931\", TimeNotSynchronized), (\"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89\", TimeNotSynchronized), (\"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005\", TimeNotSynchronized)]","latency_us":483680,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:42.692751591Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"921.280361ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8e80a662-35ae-42ab-89e2-abc60c6f21d1\", \"content-length\": \"124\", \"date\": \"Fri, 01 Mar 2024 23:16:42 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"8e80a662-35ae-42ab-89e2-abc60c6f21d1\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 720f4076-e581-438f-9cb1-67dafb6cb05c, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 1c292cfd-336a-4f75-8eb1-9f4fab725484, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bb8fa3f4-707c-4207-88cf-a0bf61d629c1, kind: Service { id: 1c292cfd-336a-4f75-8eb1-9f4fab725484 }, name: Name(\"ntp-1c292cfd-336a-4f75-8eb1-9f4fab725484\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 148, 132])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3c66b1e8-6f9c-4af5-9706-40f991312a36, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 7290f55a-1449-4cf7-88e8-036470e643e8, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 669c55e0-1c27-4089-9470-6ce2af215227, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 44640223-615f-49ba-b60a-a001dba9c022, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: bcf0a56e-6585-45f9-8061-123749957160, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 0a603215-e206-4aa0-8930-250f278a0d89, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 3af2d634-ca96-4144-9913-082ef495b783, kind: Service { id: 0a603215-e206-4aa0-8930-250f278a0d89 }, name: Name(\"external-dns-0a603215-e206-4aa0-8930-250f278a0d89\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 224, 81])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 3f949516-3930-421a-bae2-b58ff4ea0dc8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: d086f287-39ab-42aa-b1b5-4d91b0388531, kind: Service { id: 3f949516-3930-421a-bae2-b58ff4ea0dc8 }, name: Name(\"external-dns-3f949516-3930-421a-bae2-b58ff4ea0dc8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 146])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 4ea96463-fca7-4b4a-8a70-4e88cc368897, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: c6ce2abe-65af-4c28-bdee-3dca5bb68c72, kind: Service { id: 4ea96463-fca7-4b4a-8a70-4e88cc368897 }, name: Name(\"nexus-4ea96463-fca7-4b4a-8a70-4e88cc368897\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 933c0d3a-d49d-431b-8f65-51222f67e8f3, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 06ae0dac-13b1-4ef0-995e-b56e4a7d3138, kind: Service { id: 933c0d3a-d49d-431b-8f65-51222f67e8f3 }, name: Name(\"nexus-933c0d3a-d49d-431b-8f65-51222f67e8f3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 46])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 264d5dd4-3eb2-4adb-b639-f0a26ab5534d, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 9eeb19bd-897d-4d56-ae7e-bd0911ac9557, kind: Service { id: 264d5dd4-3eb2-4adb-b639-f0a26ab5534d }, name: Name(\"nexus-264d5dd4-3eb2-4adb-b639-f0a26ab5534d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 168, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: a87cb16a-062c-4865-a37d-8e0d7e2bd2b9, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 74802856-0b1c-433c-9e82-d4aedb5e0749, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ecba71f7-525f-4818-b686-c9ff63ca561f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 925e71e3-e0b8-4973-8ff8-5438a755ef84, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 282e3323-14c4-4d62-8807-8693fcbf40ea, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 3de24d1d-6129-402e-99c6-cbe1a3727280, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 7f367ce2-eb70-48d4-b9f5-2554c178c7d8, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: cf8e7366-58a5-4520-b2ad-904e641779d8, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 81f36664-5b2e-47bc-8f86-8336a864a7bb, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 5bbf29d8-d45a-494e-b486-773cde91df47, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 6c91a708-fa2b-487f-8528-4d94d9438931, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 85a1d6ba-7262-4d27-a9fc-1d1499e79005, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 056f8fb6-5500-4b9e-92d0-2be6e076a7d3, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 5fb374a2-2d09-404e-b2d8-6f14e3884eab, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:43.615490905Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 720f4076-e581-438f-9cb1-67dafb6cb05c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:43.616264004Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 720f4076-e581-438f-9cb1-67dafb6cb05c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:43.616280314Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:43.637658468Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:43.637726869Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:43.65816099Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:43.658243522Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c66b1e8-6f9c-4af5-9706-40f991312a36, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:43.677684498Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3c66b1e8-6f9c-4af5-9706-40f991312a36, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:43.67773886Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7290f55a-1449-4cf7-88e8-036470e643e8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:43.696578491Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7290f55a-1449-4cf7-88e8-036470e643e8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:43.696645663Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 669c55e0-1c27-4089-9470-6ce2af215227, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:43.714955231Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 669c55e0-1c27-4089-9470-6ce2af215227, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:43.715011983Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 44640223-615f-49ba-b60a-a001dba9c022, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:43.73286444Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 44640223-615f-49ba-b60a-a001dba9c022, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:43.732916771Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcf0a56e-6585-45f9-8061-123749957160, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:43.7508336Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bcf0a56e-6585-45f9-8061-123749957160, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:43.750900182Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a603215-e206-4aa0-8930-250f278a0d89, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:43.770445901Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0a603215-e206-4aa0-8930-250f278a0d89, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:43.770503372Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f949516-3930-421a-bae2-b58ff4ea0dc8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:43.789100838Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3f949516-3930-421a-bae2-b58ff4ea0dc8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:43.789162299Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 74802856-0b1c-433c-9e82-d4aedb5e0749, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:43.807526119Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 74802856-0b1c-433c-9e82-d4aedb5e0749, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:43.80758229Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3de24d1d-6129-402e-99c6-cbe1a3727280, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:43.825567961Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3de24d1d-6129-402e-99c6-cbe1a3727280, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:43.825631293Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f367ce2-eb70-48d4-b9f5-2554c178c7d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:43.844568596Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7f367ce2-eb70-48d4-b9f5-2554c178c7d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:43.844624348Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cf8e7366-58a5-4520-b2ad-904e641779d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:43.862862215Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cf8e7366-58a5-4520-b2ad-904e641779d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:43.862921446Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 81f36664-5b2e-47bc-8f86-8336a864a7bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:43.880995739Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 81f36664-5b2e-47bc-8f86-8336a864a7bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:43.881058761Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5bbf29d8-d45a-494e-b486-773cde91df47, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:43.899076582Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5bbf29d8-d45a-494e-b486-773cde91df47, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:43.899142144Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c91a708-fa2b-487f-8528-4d94d9438931, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:43.917304878Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6c91a708-fa2b-487f-8528-4d94d9438931, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:43.9173662Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85a1d6ba-7262-4d27-a9fc-1d1499e79005, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:43.935651408Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 85a1d6ba-7262-4d27-a9fc-1d1499e79005, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:43.935707549Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 056f8fb6-5500-4b9e-92d0-2be6e076a7d3, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:43.953933076Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 056f8fb6-5500-4b9e-92d0-2be6e076a7d3, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:43.953981827Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5fb374a2-2d09-404e-b2d8-6f14e3884eab, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:43.972538051Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5fb374a2-2d09-404e-b2d8-6f14e3884eab, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:43.972602273Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:43.990650895Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:43.990727657Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:43.990978433Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:43.991140767Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:44.084399702Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:44.084457343Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:44.084474033Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:44.084480124Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:44.084486914Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:44.084504384Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:44.084510224Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:44.084533145Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:44.084538955Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:44.084545405Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:44.084559506Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:44.084566596Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:44.084572056Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:44.084578816Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:44.084584396Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:44.084591046Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:44.084598036Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:44.084620237Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:44.084626137Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:44.08474234Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"24b261de-6f54-4645-b114-d39f5bcdd616","remote_addr":"[fd00:1122:3344:101::1]:42557","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f\", TimeNotSynchronized), (\"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb\", TimeNotSynchronized), (\"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3\", TimeNotSynchronized), (\"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab\", TimeNotSynchronized), (\"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3\", TimeNotSynchronized), (\"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84\", TimeNotSynchronized), (\"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005\", TimeNotSynchronized), (\"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8\", TimeNotSynchronized), (\"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897\", TimeNotSynchronized), (\"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea\", TimeNotSynchronized), (\"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d\", TimeNotSynchronized), (\"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749\", TimeNotSynchronized), (\"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8\", TimeNotSynchronized), (\"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9\", TimeNotSynchronized), (\"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280\", TimeNotSynchronized), (\"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47\", TimeNotSynchronized), (\"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8\", TimeNotSynchronized), (\"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931\", TimeNotSynchronized), (\"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89\", TimeNotSynchronized)]","latency_us":468676,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:44.084973256Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"1.270598024s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"24b261de-6f54-4645-b114-d39f5bcdd616\", \"content-length\": \"124\", \"date\": \"Fri, 01 Mar 2024 23:16:43 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"24b261de-6f54-4645-b114-d39f5bcdd616\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 720f4076-e581-438f-9cb1-67dafb6cb05c, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 1c292cfd-336a-4f75-8eb1-9f4fab725484, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bb8fa3f4-707c-4207-88cf-a0bf61d629c1, kind: Service { id: 1c292cfd-336a-4f75-8eb1-9f4fab725484 }, name: Name(\"ntp-1c292cfd-336a-4f75-8eb1-9f4fab725484\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 148, 132])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3c66b1e8-6f9c-4af5-9706-40f991312a36, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 7290f55a-1449-4cf7-88e8-036470e643e8, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 669c55e0-1c27-4089-9470-6ce2af215227, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 44640223-615f-49ba-b60a-a001dba9c022, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: bcf0a56e-6585-45f9-8061-123749957160, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 0a603215-e206-4aa0-8930-250f278a0d89, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 3af2d634-ca96-4144-9913-082ef495b783, kind: Service { id: 0a603215-e206-4aa0-8930-250f278a0d89 }, name: Name(\"external-dns-0a603215-e206-4aa0-8930-250f278a0d89\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 224, 81])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 3f949516-3930-421a-bae2-b58ff4ea0dc8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: d086f287-39ab-42aa-b1b5-4d91b0388531, kind: Service { id: 3f949516-3930-421a-bae2-b58ff4ea0dc8 }, name: Name(\"external-dns-3f949516-3930-421a-bae2-b58ff4ea0dc8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 146])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 4ea96463-fca7-4b4a-8a70-4e88cc368897, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: c6ce2abe-65af-4c28-bdee-3dca5bb68c72, kind: Service { id: 4ea96463-fca7-4b4a-8a70-4e88cc368897 }, name: Name(\"nexus-4ea96463-fca7-4b4a-8a70-4e88cc368897\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 933c0d3a-d49d-431b-8f65-51222f67e8f3, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 06ae0dac-13b1-4ef0-995e-b56e4a7d3138, kind: Service { id: 933c0d3a-d49d-431b-8f65-51222f67e8f3 }, name: Name(\"nexus-933c0d3a-d49d-431b-8f65-51222f67e8f3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 46])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 264d5dd4-3eb2-4adb-b639-f0a26ab5534d, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 9eeb19bd-897d-4d56-ae7e-bd0911ac9557, kind: Service { id: 264d5dd4-3eb2-4adb-b639-f0a26ab5534d }, name: Name(\"nexus-264d5dd4-3eb2-4adb-b639-f0a26ab5534d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 168, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: a87cb16a-062c-4865-a37d-8e0d7e2bd2b9, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 74802856-0b1c-433c-9e82-d4aedb5e0749, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ecba71f7-525f-4818-b686-c9ff63ca561f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 925e71e3-e0b8-4973-8ff8-5438a755ef84, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 282e3323-14c4-4d62-8807-8693fcbf40ea, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 3de24d1d-6129-402e-99c6-cbe1a3727280, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 7f367ce2-eb70-48d4-b9f5-2554c178c7d8, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: cf8e7366-58a5-4520-b2ad-904e641779d8, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 81f36664-5b2e-47bc-8f86-8336a864a7bb, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 5bbf29d8-d45a-494e-b486-773cde91df47, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 6c91a708-fa2b-487f-8528-4d94d9438931, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 85a1d6ba-7262-4d27-a9fc-1d1499e79005, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 056f8fb6-5500-4b9e-92d0-2be6e076a7d3, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 5fb374a2-2d09-404e-b2d8-6f14e3884eab, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:45.357452847Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 720f4076-e581-438f-9cb1-67dafb6cb05c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:45.358848031Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 720f4076-e581-438f-9cb1-67dafb6cb05c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:45.358864301Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:45.381438714Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:45.381483755Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:45.402949461Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:45.402986682Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c66b1e8-6f9c-4af5-9706-40f991312a36, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:45.421979087Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3c66b1e8-6f9c-4af5-9706-40f991312a36, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:45.422029298Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7290f55a-1449-4cf7-88e8-036470e643e8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:45.440718536Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7290f55a-1449-4cf7-88e8-036470e643e8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:45.440759537Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 669c55e0-1c27-4089-9470-6ce2af215227, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:45.458654216Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 669c55e0-1c27-4089-9470-6ce2af215227, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:45.458726777Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 44640223-615f-49ba-b60a-a001dba9c022, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:45.478512172Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 44640223-615f-49ba-b60a-a001dba9c022, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:45.478581154Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcf0a56e-6585-45f9-8061-123749957160, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:45.497501847Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bcf0a56e-6585-45f9-8061-123749957160, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:45.497565759Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a603215-e206-4aa0-8930-250f278a0d89, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:45.515763905Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0a603215-e206-4aa0-8930-250f278a0d89, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:45.515809306Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f949516-3930-421a-bae2-b58ff4ea0dc8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:45.534127075Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3f949516-3930-421a-bae2-b58ff4ea0dc8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:45.534187046Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 74802856-0b1c-433c-9e82-d4aedb5e0749, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:45.552536215Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 74802856-0b1c-433c-9e82-d4aedb5e0749, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:45.552591127Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3de24d1d-6129-402e-99c6-cbe1a3727280, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:45.571277085Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3de24d1d-6129-402e-99c6-cbe1a3727280, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:45.571330726Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f367ce2-eb70-48d4-b9f5-2554c178c7d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:45.589929931Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7f367ce2-eb70-48d4-b9f5-2554c178c7d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:45.589983642Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cf8e7366-58a5-4520-b2ad-904e641779d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:45.607934582Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cf8e7366-58a5-4520-b2ad-904e641779d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:45.607988483Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 81f36664-5b2e-47bc-8f86-8336a864a7bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:45.626338733Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 81f36664-5b2e-47bc-8f86-8336a864a7bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:45.626387664Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5bbf29d8-d45a-494e-b486-773cde91df47, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:45.644865987Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5bbf29d8-d45a-494e-b486-773cde91df47, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:45.644924088Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c91a708-fa2b-487f-8528-4d94d9438931, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:45.664242221Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6c91a708-fa2b-487f-8528-4d94d9438931, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:45.664294772Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85a1d6ba-7262-4d27-a9fc-1d1499e79005, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:45.683132054Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 85a1d6ba-7262-4d27-a9fc-1d1499e79005, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:45.683180675Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 056f8fb6-5500-4b9e-92d0-2be6e076a7d3, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:45.701785531Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 056f8fb6-5500-4b9e-92d0-2be6e076a7d3, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:45.701859103Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5fb374a2-2d09-404e-b2d8-6f14e3884eab, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:45.720232863Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5fb374a2-2d09-404e-b2d8-6f14e3884eab, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:45.720286294Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:45.738853119Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:45.738938571Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:45.739129165Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:45.739280199Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:45.833652661Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:45.833705382Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:45.833713503Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:45.833729373Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:45.833749803Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:45.833755874Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:45.833760324Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:45.833766754Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:45.833771274Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:45.833777654Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:45.833783354Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:45.833788675Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:45.833796175Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:45.833802885Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:45.833808145Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:45.833814775Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:45.833820495Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:45.833827325Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:45.833832556Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:45.833949998Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"7db124e8-8377-4c65-8331-9b77e7be54be","remote_addr":"[fd00:1122:3344:101::1]:42557","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47\", TimeNotSynchronized), (\"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea\", TimeNotSynchronized), (\"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931\", TimeNotSynchronized), (\"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab\", TimeNotSynchronized), (\"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb\", TimeNotSynchronized), (\"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8\", TimeNotSynchronized), (\"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3\", TimeNotSynchronized), (\"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d\", TimeNotSynchronized), (\"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749\", TimeNotSynchronized), (\"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9\", TimeNotSynchronized), (\"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005\", TimeNotSynchronized), (\"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3\", TimeNotSynchronized), (\"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8\", TimeNotSynchronized), (\"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8\", TimeNotSynchronized), (\"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89\", TimeNotSynchronized), (\"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f\", TimeNotSynchronized), (\"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84\", TimeNotSynchronized), (\"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897\", TimeNotSynchronized), (\"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280\", TimeNotSynchronized)]","latency_us":475736,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:45.834189374Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"1.101288875s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7db124e8-8377-4c65-8331-9b77e7be54be\", \"content-length\": \"124\", \"date\": \"Fri, 01 Mar 2024 23:16:45 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"7db124e8-8377-4c65-8331-9b77e7be54be\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 720f4076-e581-438f-9cb1-67dafb6cb05c, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 1c292cfd-336a-4f75-8eb1-9f4fab725484, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bb8fa3f4-707c-4207-88cf-a0bf61d629c1, kind: Service { id: 1c292cfd-336a-4f75-8eb1-9f4fab725484 }, name: Name(\"ntp-1c292cfd-336a-4f75-8eb1-9f4fab725484\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 148, 132])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3c66b1e8-6f9c-4af5-9706-40f991312a36, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 7290f55a-1449-4cf7-88e8-036470e643e8, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 669c55e0-1c27-4089-9470-6ce2af215227, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 44640223-615f-49ba-b60a-a001dba9c022, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: bcf0a56e-6585-45f9-8061-123749957160, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 0a603215-e206-4aa0-8930-250f278a0d89, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 3af2d634-ca96-4144-9913-082ef495b783, kind: Service { id: 0a603215-e206-4aa0-8930-250f278a0d89 }, name: Name(\"external-dns-0a603215-e206-4aa0-8930-250f278a0d89\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 224, 81])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 3f949516-3930-421a-bae2-b58ff4ea0dc8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: d086f287-39ab-42aa-b1b5-4d91b0388531, kind: Service { id: 3f949516-3930-421a-bae2-b58ff4ea0dc8 }, name: Name(\"external-dns-3f949516-3930-421a-bae2-b58ff4ea0dc8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 146])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 4ea96463-fca7-4b4a-8a70-4e88cc368897, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: c6ce2abe-65af-4c28-bdee-3dca5bb68c72, kind: Service { id: 4ea96463-fca7-4b4a-8a70-4e88cc368897 }, name: Name(\"nexus-4ea96463-fca7-4b4a-8a70-4e88cc368897\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 933c0d3a-d49d-431b-8f65-51222f67e8f3, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 06ae0dac-13b1-4ef0-995e-b56e4a7d3138, kind: Service { id: 933c0d3a-d49d-431b-8f65-51222f67e8f3 }, name: Name(\"nexus-933c0d3a-d49d-431b-8f65-51222f67e8f3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 46])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 264d5dd4-3eb2-4adb-b639-f0a26ab5534d, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 9eeb19bd-897d-4d56-ae7e-bd0911ac9557, kind: Service { id: 264d5dd4-3eb2-4adb-b639-f0a26ab5534d }, name: Name(\"nexus-264d5dd4-3eb2-4adb-b639-f0a26ab5534d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 168, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: a87cb16a-062c-4865-a37d-8e0d7e2bd2b9, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 74802856-0b1c-433c-9e82-d4aedb5e0749, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ecba71f7-525f-4818-b686-c9ff63ca561f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 925e71e3-e0b8-4973-8ff8-5438a755ef84, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 282e3323-14c4-4d62-8807-8693fcbf40ea, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 3de24d1d-6129-402e-99c6-cbe1a3727280, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 7f367ce2-eb70-48d4-b9f5-2554c178c7d8, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: cf8e7366-58a5-4520-b2ad-904e641779d8, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 81f36664-5b2e-47bc-8f86-8336a864a7bb, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 5bbf29d8-d45a-494e-b486-773cde91df47, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 6c91a708-fa2b-487f-8528-4d94d9438931, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 85a1d6ba-7262-4d27-a9fc-1d1499e79005, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 056f8fb6-5500-4b9e-92d0-2be6e076a7d3, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 5fb374a2-2d09-404e-b2d8-6f14e3884eab, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:46.936783923Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 720f4076-e581-438f-9cb1-67dafb6cb05c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:46.937394749Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 720f4076-e581-438f-9cb1-67dafb6cb05c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:46.937409409Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:46.958487465Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:46.958533926Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:46.979429228Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:46.979476119Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c66b1e8-6f9c-4af5-9706-40f991312a36, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:46.998572997Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3c66b1e8-6f9c-4af5-9706-40f991312a36, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:46.998619698Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7290f55a-1449-4cf7-88e8-036470e643e8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:47.017411279Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7290f55a-1449-4cf7-88e8-036470e643e8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:47.01745243Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 669c55e0-1c27-4089-9470-6ce2af215227, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:47.037615353Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 669c55e0-1c27-4089-9470-6ce2af215227, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:47.037656045Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 44640223-615f-49ba-b60a-a001dba9c022, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:47.055511242Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 44640223-615f-49ba-b60a-a001dba9c022, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:47.055554093Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcf0a56e-6585-45f9-8061-123749957160, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:47.07420989Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bcf0a56e-6585-45f9-8061-123749957160, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:47.074252991Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a603215-e206-4aa0-8930-250f278a0d89, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:47.09216898Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0a603215-e206-4aa0-8930-250f278a0d89, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:47.092213751Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f949516-3930-421a-bae2-b58ff4ea0dc8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:47.110418027Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3f949516-3930-421a-bae2-b58ff4ea0dc8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:47.110472398Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 74802856-0b1c-433c-9e82-d4aedb5e0749, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:47.129215588Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 74802856-0b1c-433c-9e82-d4aedb5e0749, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:47.129258838Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3de24d1d-6129-402e-99c6-cbe1a3727280, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:47.14728798Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3de24d1d-6129-402e-99c6-cbe1a3727280, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:47.147341881Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f367ce2-eb70-48d4-b9f5-2554c178c7d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:47.165572518Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7f367ce2-eb70-48d4-b9f5-2554c178c7d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:47.165608409Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cf8e7366-58a5-4520-b2ad-904e641779d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:47.18320944Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cf8e7366-58a5-4520-b2ad-904e641779d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:47.183248431Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 81f36664-5b2e-47bc-8f86-8336a864a7bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:47.201919958Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 81f36664-5b2e-47bc-8f86-8336a864a7bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:47.201959939Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5bbf29d8-d45a-494e-b486-773cde91df47, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:47.221944979Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5bbf29d8-d45a-494e-b486-773cde91df47, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:47.22198227Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c91a708-fa2b-487f-8528-4d94d9438931, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:47.240425752Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6c91a708-fa2b-487f-8528-4d94d9438931, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:47.240465783Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85a1d6ba-7262-4d27-a9fc-1d1499e79005, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:47.258265149Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 85a1d6ba-7262-4d27-a9fc-1d1499e79005, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:47.25831143Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 056f8fb6-5500-4b9e-92d0-2be6e076a7d3, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:47.276238239Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 056f8fb6-5500-4b9e-92d0-2be6e076a7d3, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:47.27627807Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5fb374a2-2d09-404e-b2d8-6f14e3884eab, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:47.294809724Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5fb374a2-2d09-404e-b2d8-6f14e3884eab, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:47.294849805Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:47.312526618Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:47.312573969Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:47.312780644Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:47.312920978Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:47.402036661Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:47.402134183Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:47.402143063Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:47.402158624Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:47.402166284Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:47.402175094Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:47.402184044Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:47.402192705Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:47.402208025Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:47.402214675Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:47.402229316Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:47.402236046Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:47.402244866Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:47.402253476Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:47.402262966Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:47.402278297Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:47.402288807Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:47.402294167Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:47.402303777Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:47.402372879Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"e1cd68e3-c7cc-4238-902b-22bbe4aae8df","remote_addr":"[fd00:1122:3344:101::1]:42557","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005\", TimeNotSynchronized), (\"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931\", TimeNotSynchronized), (\"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8\", TimeNotSynchronized), (\"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea\", TimeNotSynchronized), (\"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8\", TimeNotSynchronized), (\"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb\", TimeNotSynchronized), (\"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749\", TimeNotSynchronized), (\"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f\", TimeNotSynchronized), (\"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84\", TimeNotSynchronized), (\"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89\", TimeNotSynchronized), (\"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3\", TimeNotSynchronized), (\"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9\", TimeNotSynchronized), (\"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897\", TimeNotSynchronized), (\"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280\", TimeNotSynchronized), (\"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8\", TimeNotSynchronized), (\"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d\", TimeNotSynchronized), (\"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47\", TimeNotSynchronized), (\"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3\", TimeNotSynchronized), (\"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab\", TimeNotSynchronized)]","latency_us":465175,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:47.402646006Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"1.360730121s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e1cd68e3-c7cc-4238-902b-22bbe4aae8df\", \"content-length\": \"124\", \"date\": \"Fri, 01 Mar 2024 23:16:47 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e1cd68e3-c7cc-4238-902b-22bbe4aae8df\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 720f4076-e581-438f-9cb1-67dafb6cb05c, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 1c292cfd-336a-4f75-8eb1-9f4fab725484, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bb8fa3f4-707c-4207-88cf-a0bf61d629c1, kind: Service { id: 1c292cfd-336a-4f75-8eb1-9f4fab725484 }, name: Name(\"ntp-1c292cfd-336a-4f75-8eb1-9f4fab725484\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 148, 132])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3c66b1e8-6f9c-4af5-9706-40f991312a36, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 7290f55a-1449-4cf7-88e8-036470e643e8, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 669c55e0-1c27-4089-9470-6ce2af215227, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 44640223-615f-49ba-b60a-a001dba9c022, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: bcf0a56e-6585-45f9-8061-123749957160, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 0a603215-e206-4aa0-8930-250f278a0d89, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 3af2d634-ca96-4144-9913-082ef495b783, kind: Service { id: 0a603215-e206-4aa0-8930-250f278a0d89 }, name: Name(\"external-dns-0a603215-e206-4aa0-8930-250f278a0d89\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 224, 81])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 3f949516-3930-421a-bae2-b58ff4ea0dc8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: d086f287-39ab-42aa-b1b5-4d91b0388531, kind: Service { id: 3f949516-3930-421a-bae2-b58ff4ea0dc8 }, name: Name(\"external-dns-3f949516-3930-421a-bae2-b58ff4ea0dc8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 146])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 4ea96463-fca7-4b4a-8a70-4e88cc368897, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: c6ce2abe-65af-4c28-bdee-3dca5bb68c72, kind: Service { id: 4ea96463-fca7-4b4a-8a70-4e88cc368897 }, name: Name(\"nexus-4ea96463-fca7-4b4a-8a70-4e88cc368897\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 933c0d3a-d49d-431b-8f65-51222f67e8f3, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 06ae0dac-13b1-4ef0-995e-b56e4a7d3138, kind: Service { id: 933c0d3a-d49d-431b-8f65-51222f67e8f3 }, name: Name(\"nexus-933c0d3a-d49d-431b-8f65-51222f67e8f3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 46])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 264d5dd4-3eb2-4adb-b639-f0a26ab5534d, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 9eeb19bd-897d-4d56-ae7e-bd0911ac9557, kind: Service { id: 264d5dd4-3eb2-4adb-b639-f0a26ab5534d }, name: Name(\"nexus-264d5dd4-3eb2-4adb-b639-f0a26ab5534d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 168, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: a87cb16a-062c-4865-a37d-8e0d7e2bd2b9, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 74802856-0b1c-433c-9e82-d4aedb5e0749, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ecba71f7-525f-4818-b686-c9ff63ca561f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 925e71e3-e0b8-4973-8ff8-5438a755ef84, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 282e3323-14c4-4d62-8807-8693fcbf40ea, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 3de24d1d-6129-402e-99c6-cbe1a3727280, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 7f367ce2-eb70-48d4-b9f5-2554c178c7d8, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: cf8e7366-58a5-4520-b2ad-904e641779d8, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 81f36664-5b2e-47bc-8f86-8336a864a7bb, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 5bbf29d8-d45a-494e-b486-773cde91df47, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 6c91a708-fa2b-487f-8528-4d94d9438931, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 85a1d6ba-7262-4d27-a9fc-1d1499e79005, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 056f8fb6-5500-4b9e-92d0-2be6e076a7d3, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 5fb374a2-2d09-404e-b2d8-6f14e3884eab, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:48.764384372Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 720f4076-e581-438f-9cb1-67dafb6cb05c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:48.765119381Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 720f4076-e581-438f-9cb1-67dafb6cb05c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:48.765137691Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:48.787736015Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:48.787808206Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:48.80918394Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:48.809247742Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c66b1e8-6f9c-4af5-9706-40f991312a36, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:48.828513483Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3c66b1e8-6f9c-4af5-9706-40f991312a36, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:48.828579685Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7290f55a-1449-4cf7-88e8-036470e643e8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:48.846684948Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7290f55a-1449-4cf7-88e8-036470e643e8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:48.8467522Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 669c55e0-1c27-4089-9470-6ce2af215227, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:48.86510137Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 669c55e0-1c27-4089-9470-6ce2af215227, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:48.865155981Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 44640223-615f-49ba-b60a-a001dba9c022, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:48.885205502Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 44640223-615f-49ba-b60a-a001dba9c022, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:48.885262273Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcf0a56e-6585-45f9-8061-123749957160, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:48.903146592Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bcf0a56e-6585-45f9-8061-123749957160, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:48.903203763Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a603215-e206-4aa0-8930-250f278a0d89, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:48.92144218Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0a603215-e206-4aa0-8930-250f278a0d89, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:48.921491361Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f949516-3930-421a-bae2-b58ff4ea0dc8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:48.940027355Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3f949516-3930-421a-bae2-b58ff4ea0dc8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:48.940089847Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 74802856-0b1c-433c-9e82-d4aedb5e0749, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:48.958976679Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 74802856-0b1c-433c-9e82-d4aedb5e0749, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:48.959029131Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3de24d1d-6129-402e-99c6-cbe1a3727280, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:48.977804851Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3de24d1d-6129-402e-99c6-cbe1a3727280, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:48.977877812Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f367ce2-eb70-48d4-b9f5-2554c178c7d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:48.995583346Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7f367ce2-eb70-48d4-b9f5-2554c178c7d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:48.995646518Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cf8e7366-58a5-4520-b2ad-904e641779d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:49.014716965Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cf8e7366-58a5-4520-b2ad-904e641779d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:49.014786486Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 81f36664-5b2e-47bc-8f86-8336a864a7bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:49.035169786Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 81f36664-5b2e-47bc-8f86-8336a864a7bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:49.035219257Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5bbf29d8-d45a-494e-b486-773cde91df47, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:49.053503525Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5bbf29d8-d45a-494e-b486-773cde91df47, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:49.053564476Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c91a708-fa2b-487f-8528-4d94d9438931, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:49.073253059Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6c91a708-fa2b-487f-8528-4d94d9438931, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:49.07331168Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85a1d6ba-7262-4d27-a9fc-1d1499e79005, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:49.092763707Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 85a1d6ba-7262-4d27-a9fc-1d1499e79005, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:49.092820588Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 056f8fb6-5500-4b9e-92d0-2be6e076a7d3, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:49.111181038Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 056f8fb6-5500-4b9e-92d0-2be6e076a7d3, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:49.11124524Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5fb374a2-2d09-404e-b2d8-6f14e3884eab, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:49.130488231Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5fb374a2-2d09-404e-b2d8-6f14e3884eab, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:49.130543902Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:49.14962262Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:49.149687571Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:49.149974918Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:49.150121052Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:49.242475214Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:49.242543556Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:49.242552246Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:49.242558106Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:49.242565126Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:49.242571877Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:49.242583167Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:49.242590037Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:49.242595887Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:49.242602617Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:49.242613347Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:49.242619608Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:49.242625098Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:49.242640328Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:49.242647399Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:49.242652928Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:49.242664909Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:49.242671559Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:49.242676829Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:49.242822003Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"971fbd36-ec88-4b01-9583-254b05dfc6fd","remote_addr":"[fd00:1122:3344:101::1]:42557","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9\", TimeNotSynchronized), (\"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931\", TimeNotSynchronized), (\"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3\", TimeNotSynchronized), (\"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749\", TimeNotSynchronized), (\"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89\", TimeNotSynchronized), (\"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3\", TimeNotSynchronized), (\"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f\", TimeNotSynchronized), (\"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84\", TimeNotSynchronized), (\"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab\", TimeNotSynchronized), (\"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d\", TimeNotSynchronized), (\"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280\", TimeNotSynchronized), (\"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8\", TimeNotSynchronized), (\"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8\", TimeNotSynchronized), (\"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea\", TimeNotSynchronized), (\"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897\", TimeNotSynchronized), (\"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8\", TimeNotSynchronized), (\"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47\", TimeNotSynchronized), (\"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb\", TimeNotSynchronized), (\"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005\", TimeNotSynchronized)]","latency_us":477924,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:49.243078609Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"1.803748701s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"971fbd36-ec88-4b01-9583-254b05dfc6fd\", \"content-length\": \"124\", \"date\": \"Fri, 01 Mar 2024 23:16:48 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"971fbd36-ec88-4b01-9583-254b05dfc6fd\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 720f4076-e581-438f-9cb1-67dafb6cb05c, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 1c292cfd-336a-4f75-8eb1-9f4fab725484, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bb8fa3f4-707c-4207-88cf-a0bf61d629c1, kind: Service { id: 1c292cfd-336a-4f75-8eb1-9f4fab725484 }, name: Name(\"ntp-1c292cfd-336a-4f75-8eb1-9f4fab725484\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 148, 132])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3c66b1e8-6f9c-4af5-9706-40f991312a36, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 7290f55a-1449-4cf7-88e8-036470e643e8, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 669c55e0-1c27-4089-9470-6ce2af215227, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 44640223-615f-49ba-b60a-a001dba9c022, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: bcf0a56e-6585-45f9-8061-123749957160, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 0a603215-e206-4aa0-8930-250f278a0d89, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 3af2d634-ca96-4144-9913-082ef495b783, kind: Service { id: 0a603215-e206-4aa0-8930-250f278a0d89 }, name: Name(\"external-dns-0a603215-e206-4aa0-8930-250f278a0d89\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 224, 81])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 3f949516-3930-421a-bae2-b58ff4ea0dc8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: d086f287-39ab-42aa-b1b5-4d91b0388531, kind: Service { id: 3f949516-3930-421a-bae2-b58ff4ea0dc8 }, name: Name(\"external-dns-3f949516-3930-421a-bae2-b58ff4ea0dc8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 146])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 4ea96463-fca7-4b4a-8a70-4e88cc368897, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: c6ce2abe-65af-4c28-bdee-3dca5bb68c72, kind: Service { id: 4ea96463-fca7-4b4a-8a70-4e88cc368897 }, name: Name(\"nexus-4ea96463-fca7-4b4a-8a70-4e88cc368897\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 933c0d3a-d49d-431b-8f65-51222f67e8f3, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 06ae0dac-13b1-4ef0-995e-b56e4a7d3138, kind: Service { id: 933c0d3a-d49d-431b-8f65-51222f67e8f3 }, name: Name(\"nexus-933c0d3a-d49d-431b-8f65-51222f67e8f3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 46])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 264d5dd4-3eb2-4adb-b639-f0a26ab5534d, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 9eeb19bd-897d-4d56-ae7e-bd0911ac9557, kind: Service { id: 264d5dd4-3eb2-4adb-b639-f0a26ab5534d }, name: Name(\"nexus-264d5dd4-3eb2-4adb-b639-f0a26ab5534d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 168, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: a87cb16a-062c-4865-a37d-8e0d7e2bd2b9, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 74802856-0b1c-433c-9e82-d4aedb5e0749, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ecba71f7-525f-4818-b686-c9ff63ca561f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 925e71e3-e0b8-4973-8ff8-5438a755ef84, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 282e3323-14c4-4d62-8807-8693fcbf40ea, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 3de24d1d-6129-402e-99c6-cbe1a3727280, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 7f367ce2-eb70-48d4-b9f5-2554c178c7d8, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: cf8e7366-58a5-4520-b2ad-904e641779d8, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 81f36664-5b2e-47bc-8f86-8336a864a7bb, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 5bbf29d8-d45a-494e-b486-773cde91df47, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 6c91a708-fa2b-487f-8528-4d94d9438931, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 85a1d6ba-7262-4d27-a9fc-1d1499e79005, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 056f8fb6-5500-4b9e-92d0-2be6e076a7d3, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 5fb374a2-2d09-404e-b2d8-6f14e3884eab, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:51.048024404Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 720f4076-e581-438f-9cb1-67dafb6cb05c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:51.048846144Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 720f4076-e581-438f-9cb1-67dafb6cb05c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:51.048861914Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:51.074065582Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:51.074119753Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:51.095665601Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:51.095726792Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c66b1e8-6f9c-4af5-9706-40f991312a36, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:51.116912321Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3c66b1e8-6f9c-4af5-9706-40f991312a36, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:51.116965913Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7290f55a-1449-4cf7-88e8-036470e643e8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:51.136284166Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7290f55a-1449-4cf7-88e8-036470e643e8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:51.136332217Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 669c55e0-1c27-4089-9470-6ce2af215227, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:51.154755868Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 669c55e0-1c27-4089-9470-6ce2af215227, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:51.1548101Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 44640223-615f-49ba-b60a-a001dba9c022, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:51.172955224Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 44640223-615f-49ba-b60a-a001dba9c022, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:51.173000955Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcf0a56e-6585-45f9-8061-123749957160, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:51.192202416Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bcf0a56e-6585-45f9-8061-123749957160, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:51.192240186Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a603215-e206-4aa0-8930-250f278a0d89, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:51.211323124Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0a603215-e206-4aa0-8930-250f278a0d89, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:51.211364235Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f949516-3930-421a-bae2-b58ff4ea0dc8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:51.229027638Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3f949516-3930-421a-bae2-b58ff4ea0dc8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:51.229066239Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 74802856-0b1c-433c-9e82-d4aedb5e0749, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:51.248120135Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 74802856-0b1c-433c-9e82-d4aedb5e0749, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:51.248157056Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3de24d1d-6129-402e-99c6-cbe1a3727280, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:51.265953512Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3de24d1d-6129-402e-99c6-cbe1a3727280, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:51.265989793Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f367ce2-eb70-48d4-b9f5-2554c178c7d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:51.284411064Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7f367ce2-eb70-48d4-b9f5-2554c178c7d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:51.284465536Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cf8e7366-58a5-4520-b2ad-904e641779d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:51.302321543Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cf8e7366-58a5-4520-b2ad-904e641779d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:51.302356714Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 81f36664-5b2e-47bc-8f86-8336a864a7bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:51.320539079Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 81f36664-5b2e-47bc-8f86-8336a864a7bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:51.32057779Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5bbf29d8-d45a-494e-b486-773cde91df47, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:51.338637843Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5bbf29d8-d45a-494e-b486-773cde91df47, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:51.338677594Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c91a708-fa2b-487f-8528-4d94d9438931, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:51.356407098Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6c91a708-fa2b-487f-8528-4d94d9438931, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:51.356451719Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85a1d6ba-7262-4d27-a9fc-1d1499e79005, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:51.374175493Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 85a1d6ba-7262-4d27-a9fc-1d1499e79005, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:51.374215204Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 056f8fb6-5500-4b9e-92d0-2be6e076a7d3, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:51.391807785Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 056f8fb6-5500-4b9e-92d0-2be6e076a7d3, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:51.391851176Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5fb374a2-2d09-404e-b2d8-6f14e3884eab, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:51.410080803Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5fb374a2-2d09-404e-b2d8-6f14e3884eab, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:51.410124234Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:51.428598316Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:51.428654908Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:51.428823982Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:51.428947605Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:51.523233315Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:51.523298186Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:51.523307007Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:51.523316397Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:51.523323157Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:51.523330107Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:51.523345657Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:51.523362148Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:51.523370998Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:51.523379638Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:51.523405049Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:51.523412539Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:51.523428939Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:51.52343598Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:51.52344454Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:51.52345293Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:51.52346158Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:51.52346662Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:51.52347276Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:51.523554742Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"b1e8d32f-8583-4850-8ebc-efe589581a39","remote_addr":"[fd00:1122:3344:101::1]:42557","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8\", TimeNotSynchronized), (\"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea\", TimeNotSynchronized), (\"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47\", TimeNotSynchronized), (\"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb\", TimeNotSynchronized), (\"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89\", TimeNotSynchronized), (\"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3\", TimeNotSynchronized), (\"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749\", TimeNotSynchronized), (\"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84\", TimeNotSynchronized), (\"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8\", TimeNotSynchronized), (\"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9\", TimeNotSynchronized), (\"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f\", TimeNotSynchronized), (\"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d\", TimeNotSynchronized), (\"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897\", TimeNotSynchronized), (\"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280\", TimeNotSynchronized), (\"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8\", TimeNotSynchronized), (\"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931\", TimeNotSynchronized), (\"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005\", TimeNotSynchronized), (\"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab\", TimeNotSynchronized), (\"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3\", TimeNotSynchronized)]","latency_us":474971,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:51.523821979Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"2.873363715s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b1e8d32f-8583-4850-8ebc-efe589581a39\", \"content-length\": \"124\", \"date\": \"Fri, 01 Mar 2024 23:16:51 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b1e8d32f-8583-4850-8ebc-efe589581a39\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 720f4076-e581-438f-9cb1-67dafb6cb05c, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 1c292cfd-336a-4f75-8eb1-9f4fab725484, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bb8fa3f4-707c-4207-88cf-a0bf61d629c1, kind: Service { id: 1c292cfd-336a-4f75-8eb1-9f4fab725484 }, name: Name(\"ntp-1c292cfd-336a-4f75-8eb1-9f4fab725484\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 148, 132])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3c66b1e8-6f9c-4af5-9706-40f991312a36, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 7290f55a-1449-4cf7-88e8-036470e643e8, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 669c55e0-1c27-4089-9470-6ce2af215227, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 44640223-615f-49ba-b60a-a001dba9c022, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: bcf0a56e-6585-45f9-8061-123749957160, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 0a603215-e206-4aa0-8930-250f278a0d89, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 3af2d634-ca96-4144-9913-082ef495b783, kind: Service { id: 0a603215-e206-4aa0-8930-250f278a0d89 }, name: Name(\"external-dns-0a603215-e206-4aa0-8930-250f278a0d89\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 224, 81])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 3f949516-3930-421a-bae2-b58ff4ea0dc8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: d086f287-39ab-42aa-b1b5-4d91b0388531, kind: Service { id: 3f949516-3930-421a-bae2-b58ff4ea0dc8 }, name: Name(\"external-dns-3f949516-3930-421a-bae2-b58ff4ea0dc8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 146])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 4ea96463-fca7-4b4a-8a70-4e88cc368897, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: c6ce2abe-65af-4c28-bdee-3dca5bb68c72, kind: Service { id: 4ea96463-fca7-4b4a-8a70-4e88cc368897 }, name: Name(\"nexus-4ea96463-fca7-4b4a-8a70-4e88cc368897\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 933c0d3a-d49d-431b-8f65-51222f67e8f3, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 06ae0dac-13b1-4ef0-995e-b56e4a7d3138, kind: Service { id: 933c0d3a-d49d-431b-8f65-51222f67e8f3 }, name: Name(\"nexus-933c0d3a-d49d-431b-8f65-51222f67e8f3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 46])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 264d5dd4-3eb2-4adb-b639-f0a26ab5534d, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 9eeb19bd-897d-4d56-ae7e-bd0911ac9557, kind: Service { id: 264d5dd4-3eb2-4adb-b639-f0a26ab5534d }, name: Name(\"nexus-264d5dd4-3eb2-4adb-b639-f0a26ab5534d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 168, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: a87cb16a-062c-4865-a37d-8e0d7e2bd2b9, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 74802856-0b1c-433c-9e82-d4aedb5e0749, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ecba71f7-525f-4818-b686-c9ff63ca561f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 925e71e3-e0b8-4973-8ff8-5438a755ef84, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 282e3323-14c4-4d62-8807-8693fcbf40ea, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 3de24d1d-6129-402e-99c6-cbe1a3727280, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 7f367ce2-eb70-48d4-b9f5-2554c178c7d8, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: cf8e7366-58a5-4520-b2ad-904e641779d8, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 81f36664-5b2e-47bc-8f86-8336a864a7bb, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 5bbf29d8-d45a-494e-b486-773cde91df47, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 6c91a708-fa2b-487f-8528-4d94d9438931, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 85a1d6ba-7262-4d27-a9fc-1d1499e79005, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 056f8fb6-5500-4b9e-92d0-2be6e076a7d3, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 5fb374a2-2d09-404e-b2d8-6f14e3884eab, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:54.397918963Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 720f4076-e581-438f-9cb1-67dafb6cb05c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:54.398668621Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 720f4076-e581-438f-9cb1-67dafb6cb05c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:54.398685362Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:54.41944895Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:54.419505462Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:54.440060615Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:54.440123007Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c66b1e8-6f9c-4af5-9706-40f991312a36, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:54.462212258Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3c66b1e8-6f9c-4af5-9706-40f991312a36, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:54.462272839Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7290f55a-1449-4cf7-88e8-036470e643e8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:54.483492599Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7290f55a-1449-4cf7-88e8-036470e643e8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:54.483562951Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 669c55e0-1c27-4089-9470-6ce2af215227, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:54.504543535Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 669c55e0-1c27-4089-9470-6ce2af215227, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:54.504600096Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 44640223-615f-49ba-b60a-a001dba9c022, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:54.528864581Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 44640223-615f-49ba-b60a-a001dba9c022, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:54.528928862Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcf0a56e-6585-45f9-8061-123749957160, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:54.549520847Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bcf0a56e-6585-45f9-8061-123749957160, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:54.549583178Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a603215-e206-4aa0-8930-250f278a0d89, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:54.569999118Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0a603215-e206-4aa0-8930-250f278a0d89, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:54.57005976Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f949516-3930-421a-bae2-b58ff4ea0dc8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:54.59053303Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3f949516-3930-421a-bae2-b58ff4ea0dc8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:54.590592722Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 74802856-0b1c-433c-9e82-d4aedb5e0749, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:54.610607552Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 74802856-0b1c-433c-9e82-d4aedb5e0749, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:54.610665774Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3de24d1d-6129-402e-99c6-cbe1a3727280, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:54.629854634Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3de24d1d-6129-402e-99c6-cbe1a3727280, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:54.629912805Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f367ce2-eb70-48d4-b9f5-2554c178c7d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:54.64929692Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7f367ce2-eb70-48d4-b9f5-2554c178c7d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:54.649357851Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cf8e7366-58a5-4520-b2ad-904e641779d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:54.668125361Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cf8e7366-58a5-4520-b2ad-904e641779d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:54.668172602Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 81f36664-5b2e-47bc-8f86-8336a864a7bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:54.687780393Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 81f36664-5b2e-47bc-8f86-8336a864a7bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:54.687843404Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5bbf29d8-d45a-494e-b486-773cde91df47, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:54.706822489Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5bbf29d8-d45a-494e-b486-773cde91df47, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:54.706885751Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c91a708-fa2b-487f-8528-4d94d9438931, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:54.72645258Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6c91a708-fa2b-487f-8528-4d94d9438931, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:54.726506271Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85a1d6ba-7262-4d27-a9fc-1d1499e79005, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:54.747348592Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 85a1d6ba-7262-4d27-a9fc-1d1499e79005, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:54.747409453Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 056f8fb6-5500-4b9e-92d0-2be6e076a7d3, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:54.766806459Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 056f8fb6-5500-4b9e-92d0-2be6e076a7d3, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:54.7668651Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5fb374a2-2d09-404e-b2d8-6f14e3884eab, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:54.787158927Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5fb374a2-2d09-404e-b2d8-6f14e3884eab, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:54.787220749Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:54.807126116Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:54.807198958Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:54.807408063Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:54.807547277Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:54.903455466Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:54.903552498Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:54.903561539Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:54.903568339Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:54.903574919Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:54.903581199Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:54.903592399Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:54.90360026Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:54.90361516Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:54.90362392Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:54.90363131Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:54.903645571Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:54.903652251Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:54.903659951Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:54.903665881Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:54.903671901Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:54.903685452Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:54.903691802Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:54.903699322Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:54.903804485Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"59b9491a-2da2-4952-bea5-806a74a39b9a","remote_addr":"[fd00:1122:3344:101::1]:42557","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3\", TimeNotSynchronized), (\"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897\", TimeNotSynchronized), (\"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931\", TimeNotSynchronized), (\"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3\", TimeNotSynchronized), (\"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab\", TimeNotSynchronized), (\"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9\", TimeNotSynchronized), (\"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f\", TimeNotSynchronized), (\"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8\", TimeNotSynchronized), (\"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8\", TimeNotSynchronized), (\"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb\", TimeNotSynchronized), (\"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8\", TimeNotSynchronized), (\"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea\", TimeNotSynchronized), (\"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005\", TimeNotSynchronized), (\"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749\", TimeNotSynchronized), (\"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84\", TimeNotSynchronized), (\"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280\", TimeNotSynchronized), (\"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89\", TimeNotSynchronized), (\"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d\", TimeNotSynchronized), (\"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47\", TimeNotSynchronized)]","latency_us":505340,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:54.904100532Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"2.379172454s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"59b9491a-2da2-4952-bea5-806a74a39b9a\", \"content-length\": \"124\", \"date\": \"Fri, 01 Mar 2024 23:16:54 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"59b9491a-2da2-4952-bea5-806a74a39b9a\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 720f4076-e581-438f-9cb1-67dafb6cb05c, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 1c292cfd-336a-4f75-8eb1-9f4fab725484, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bb8fa3f4-707c-4207-88cf-a0bf61d629c1, kind: Service { id: 1c292cfd-336a-4f75-8eb1-9f4fab725484 }, name: Name(\"ntp-1c292cfd-336a-4f75-8eb1-9f4fab725484\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 148, 132])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3c66b1e8-6f9c-4af5-9706-40f991312a36, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 7290f55a-1449-4cf7-88e8-036470e643e8, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 669c55e0-1c27-4089-9470-6ce2af215227, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 44640223-615f-49ba-b60a-a001dba9c022, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: bcf0a56e-6585-45f9-8061-123749957160, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 0a603215-e206-4aa0-8930-250f278a0d89, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 3af2d634-ca96-4144-9913-082ef495b783, kind: Service { id: 0a603215-e206-4aa0-8930-250f278a0d89 }, name: Name(\"external-dns-0a603215-e206-4aa0-8930-250f278a0d89\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 224, 81])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 3f949516-3930-421a-bae2-b58ff4ea0dc8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: d086f287-39ab-42aa-b1b5-4d91b0388531, kind: Service { id: 3f949516-3930-421a-bae2-b58ff4ea0dc8 }, name: Name(\"external-dns-3f949516-3930-421a-bae2-b58ff4ea0dc8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 146])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 4ea96463-fca7-4b4a-8a70-4e88cc368897, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: c6ce2abe-65af-4c28-bdee-3dca5bb68c72, kind: Service { id: 4ea96463-fca7-4b4a-8a70-4e88cc368897 }, name: Name(\"nexus-4ea96463-fca7-4b4a-8a70-4e88cc368897\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 933c0d3a-d49d-431b-8f65-51222f67e8f3, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 06ae0dac-13b1-4ef0-995e-b56e4a7d3138, kind: Service { id: 933c0d3a-d49d-431b-8f65-51222f67e8f3 }, name: Name(\"nexus-933c0d3a-d49d-431b-8f65-51222f67e8f3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 46])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 264d5dd4-3eb2-4adb-b639-f0a26ab5534d, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 9eeb19bd-897d-4d56-ae7e-bd0911ac9557, kind: Service { id: 264d5dd4-3eb2-4adb-b639-f0a26ab5534d }, name: Name(\"nexus-264d5dd4-3eb2-4adb-b639-f0a26ab5534d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 168, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: a87cb16a-062c-4865-a37d-8e0d7e2bd2b9, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 74802856-0b1c-433c-9e82-d4aedb5e0749, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ecba71f7-525f-4818-b686-c9ff63ca561f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 925e71e3-e0b8-4973-8ff8-5438a755ef84, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 282e3323-14c4-4d62-8807-8693fcbf40ea, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 3de24d1d-6129-402e-99c6-cbe1a3727280, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 7f367ce2-eb70-48d4-b9f5-2554c178c7d8, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: cf8e7366-58a5-4520-b2ad-904e641779d8, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 81f36664-5b2e-47bc-8f86-8336a864a7bb, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 5bbf29d8-d45a-494e-b486-773cde91df47, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 6c91a708-fa2b-487f-8528-4d94d9438931, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 85a1d6ba-7262-4d27-a9fc-1d1499e79005, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 056f8fb6-5500-4b9e-92d0-2be6e076a7d3, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 5fb374a2-2d09-404e-b2d8-6f14e3884eab, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:57.284249516Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 720f4076-e581-438f-9cb1-67dafb6cb05c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:57.285004375Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 720f4076-e581-438f-9cb1-67dafb6cb05c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:57.285028106Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:57.306222575Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:57.306280386Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:57.325891747Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:57.325953978Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c66b1e8-6f9c-4af5-9706-40f991312a36, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:57.345045876Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3c66b1e8-6f9c-4af5-9706-40f991312a36, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:57.345120448Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7290f55a-1449-4cf7-88e8-036470e643e8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:57.363127709Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7290f55a-1449-4cf7-88e8-036470e643e8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:57.363194961Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 669c55e0-1c27-4089-9470-6ce2af215227, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:57.381760275Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 669c55e0-1c27-4089-9470-6ce2af215227, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:57.381835187Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 44640223-615f-49ba-b60a-a001dba9c022, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:57.399362196Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 44640223-615f-49ba-b60a-a001dba9c022, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:57.399420628Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcf0a56e-6585-45f9-8061-123749957160, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:57.417557502Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bcf0a56e-6585-45f9-8061-123749957160, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:57.417613134Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a603215-e206-4aa0-8930-250f278a0d89, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:57.435905632Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0a603215-e206-4aa0-8930-250f278a0d89, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:57.435967733Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f949516-3930-421a-bae2-b58ff4ea0dc8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:57.45459244Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3f949516-3930-421a-bae2-b58ff4ea0dc8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:57.454644981Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 74802856-0b1c-433c-9e82-d4aedb5e0749, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:57.475499161Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 74802856-0b1c-433c-9e82-d4aedb5e0749, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:57.475543723Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3de24d1d-6129-402e-99c6-cbe1a3727280, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:57.495294906Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3de24d1d-6129-402e-99c6-cbe1a3727280, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:57.495366558Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f367ce2-eb70-48d4-b9f5-2554c178c7d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:57.514853086Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7f367ce2-eb70-48d4-b9f5-2554c178c7d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:57.514911637Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cf8e7366-58a5-4520-b2ad-904e641779d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:57.533574864Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cf8e7366-58a5-4520-b2ad-904e641779d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:57.533633186Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 81f36664-5b2e-47bc-8f86-8336a864a7bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:57.553462811Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 81f36664-5b2e-47bc-8f86-8336a864a7bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:57.553516603Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5bbf29d8-d45a-494e-b486-773cde91df47, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:57.57218688Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5bbf29d8-d45a-494e-b486-773cde91df47, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:57.572258502Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c91a708-fa2b-487f-8528-4d94d9438931, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:57.591253636Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6c91a708-fa2b-487f-8528-4d94d9438931, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:57.591308308Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85a1d6ba-7262-4d27-a9fc-1d1499e79005, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:57.608900139Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 85a1d6ba-7262-4d27-a9fc-1d1499e79005, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:57.60895158Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 056f8fb6-5500-4b9e-92d0-2be6e076a7d3, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:57.62731341Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 056f8fb6-5500-4b9e-92d0-2be6e076a7d3, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:57.627373771Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5fb374a2-2d09-404e-b2d8-6f14e3884eab, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:57.646447088Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5fb374a2-2d09-404e-b2d8-6f14e3884eab, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:57.64650356Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:57.665242139Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:57.66531219Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:57.665503195Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:57.665646419Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:57.757492089Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:57.75754702Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:57.75756174Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:57.75756869Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:57.757580741Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:57.757594301Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:57.757600291Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:57.757606041Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:57.757611662Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:57.757617352Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:57.757624092Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:57.757632862Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:57.757640422Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:57.757646222Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:57.757659033Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:57.757665583Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:57.757672353Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:57.757679393Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:57.757686283Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2832","zone":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:16:57.757787186Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"3798ef72-f23e-4456-b2d8-a9fdeb1f826f","remote_addr":"[fd00:1122:3344:101::1]:42557","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005\", TimeNotSynchronized), (\"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3\", TimeNotSynchronized), (\"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab\", TimeNotSynchronized), (\"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84\", TimeNotSynchronized), (\"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea\", TimeNotSynchronized), (\"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897\", TimeNotSynchronized), (\"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89\", TimeNotSynchronized), (\"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8\", TimeNotSynchronized), (\"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d\", TimeNotSynchronized), (\"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9\", TimeNotSynchronized), (\"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749\", TimeNotSynchronized), (\"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280\", TimeNotSynchronized), (\"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f\", TimeNotSynchronized), (\"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8\", TimeNotSynchronized), (\"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3\", TimeNotSynchronized), (\"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb\", TimeNotSynchronized), (\"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47\", TimeNotSynchronized), (\"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931\", TimeNotSynchronized), (\"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8\", TimeNotSynchronized)]","latency_us":473014,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:16:57.758031212Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"3.329985145s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3798ef72-f23e-4456-b2d8-a9fdeb1f826f\", \"content-length\": \"124\", \"date\": \"Fri, 01 Mar 2024 23:16:57 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3798ef72-f23e-4456-b2d8-a9fdeb1f826f\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 720f4076-e581-438f-9cb1-67dafb6cb05c, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 1c292cfd-336a-4f75-8eb1-9f4fab725484, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bb8fa3f4-707c-4207-88cf-a0bf61d629c1, kind: Service { id: 1c292cfd-336a-4f75-8eb1-9f4fab725484 }, name: Name(\"ntp-1c292cfd-336a-4f75-8eb1-9f4fab725484\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 148, 132])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3c66b1e8-6f9c-4af5-9706-40f991312a36, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 7290f55a-1449-4cf7-88e8-036470e643e8, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 669c55e0-1c27-4089-9470-6ce2af215227, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 44640223-615f-49ba-b60a-a001dba9c022, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: bcf0a56e-6585-45f9-8061-123749957160, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 0a603215-e206-4aa0-8930-250f278a0d89, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 3af2d634-ca96-4144-9913-082ef495b783, kind: Service { id: 0a603215-e206-4aa0-8930-250f278a0d89 }, name: Name(\"external-dns-0a603215-e206-4aa0-8930-250f278a0d89\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 224, 81])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 3f949516-3930-421a-bae2-b58ff4ea0dc8, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: d086f287-39ab-42aa-b1b5-4d91b0388531, kind: Service { id: 3f949516-3930-421a-bae2-b58ff4ea0dc8 }, name: Name(\"external-dns-3f949516-3930-421a-bae2-b58ff4ea0dc8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 146])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 4ea96463-fca7-4b4a-8a70-4e88cc368897, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: c6ce2abe-65af-4c28-bdee-3dca5bb68c72, kind: Service { id: 4ea96463-fca7-4b4a-8a70-4e88cc368897 }, name: Name(\"nexus-4ea96463-fca7-4b4a-8a70-4e88cc368897\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 933c0d3a-d49d-431b-8f65-51222f67e8f3, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 06ae0dac-13b1-4ef0-995e-b56e4a7d3138, kind: Service { id: 933c0d3a-d49d-431b-8f65-51222f67e8f3 }, name: Name(\"nexus-933c0d3a-d49d-431b-8f65-51222f67e8f3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 46])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 264d5dd4-3eb2-4adb-b639-f0a26ab5534d, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 9eeb19bd-897d-4d56-ae7e-bd0911ac9557, kind: Service { id: 264d5dd4-3eb2-4adb-b639-f0a26ab5534d }, name: Name(\"nexus-264d5dd4-3eb2-4adb-b639-f0a26ab5534d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 168, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: a87cb16a-062c-4865-a37d-8e0d7e2bd2b9, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 74802856-0b1c-433c-9e82-d4aedb5e0749, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: ecba71f7-525f-4818-b686-c9ff63ca561f, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 925e71e3-e0b8-4973-8ff8-5438a755ef84, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 282e3323-14c4-4d62-8807-8693fcbf40ea, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 3de24d1d-6129-402e-99c6-cbe1a3727280, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 7f367ce2-eb70-48d4-b9f5-2554c178c7d8, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: cf8e7366-58a5-4520-b2ad-904e641779d8, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 81f36664-5b2e-47bc-8f86-8336a864a7bb, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: 5bbf29d8-d45a-494e-b486-773cde91df47, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 6c91a708-fa2b-487f-8528-4d94d9438931, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 85a1d6ba-7262-4d27-a9fc-1d1499e79005, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 056f8fb6-5500-4b9e-92d0-2be6e076a7d3, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 5fb374a2-2d09-404e-b2d8-6f14e3884eab, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.089187463Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 720f4076-e581-438f-9cb1-67dafb6cb05c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.089851309Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 720f4076-e581-438f-9cb1-67dafb6cb05c, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.089865639Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.112037343Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0fb91602-87a5-4c5e-baa6-f0c1b28cef23, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.112084104Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.129636894Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2698336d-5a4b-4958-8e5d-8a26f208fcce, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.129679755Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c66b1e8-6f9c-4af5-9706-40f991312a36, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.151877629Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3c66b1e8-6f9c-4af5-9706-40f991312a36, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.15192335Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7290f55a-1449-4cf7-88e8-036470e643e8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.171340066Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7290f55a-1449-4cf7-88e8-036470e643e8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.171377326Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 669c55e0-1c27-4089-9470-6ce2af215227, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.189964042Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 669c55e0-1c27-4089-9470-6ce2af215227, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.189998772Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 44640223-615f-49ba-b60a-a001dba9c022, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.210397542Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 44640223-615f-49ba-b60a-a001dba9c022, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.210433523Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bcf0a56e-6585-45f9-8061-123749957160, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.002134812Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bcf0a56e-6585-45f9-8061-123749957160, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.002168353Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.004315606Z","hostname":"buskin","pid":1671,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:776"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a603215-e206-4aa0-8930-250f278a0d89, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.020763219Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 0a603215-e206-4aa0-8930-250f278a0d89, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.020793839Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f949516-3930-421a-bae2-b58ff4ea0dc8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.039394345Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3f949516-3930-421a-bae2-b58ff4ea0dc8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.039429086Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 74802856-0b1c-433c-9e82-d4aedb5e0749, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.058744929Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 74802856-0b1c-433c-9e82-d4aedb5e0749, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.05878773Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3de24d1d-6129-402e-99c6-cbe1a3727280, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.077887948Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 3de24d1d-6129-402e-99c6-cbe1a3727280, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.077921259Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f367ce2-eb70-48d4-b9f5-2554c178c7d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.096192086Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7f367ce2-eb70-48d4-b9f5-2554c178c7d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.096223917Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cf8e7366-58a5-4520-b2ad-904e641779d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.114499365Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: cf8e7366-58a5-4520-b2ad-904e641779d8, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.114532946Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 81f36664-5b2e-47bc-8f86-8336a864a7bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.132588348Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 81f36664-5b2e-47bc-8f86-8336a864a7bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.132632609Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5bbf29d8-d45a-494e-b486-773cde91df47, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.150555428Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5bbf29d8-d45a-494e-b486-773cde91df47, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.150592909Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c91a708-fa2b-487f-8528-4d94d9438931, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.168573669Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6c91a708-fa2b-487f-8528-4d94d9438931, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.168613301Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85a1d6ba-7262-4d27-a9fc-1d1499e79005, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.187060062Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 85a1d6ba-7262-4d27-a9fc-1d1499e79005, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.187103293Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 056f8fb6-5500-4b9e-92d0-2be6e076a7d3, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.206844897Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 056f8fb6-5500-4b9e-92d0-2be6e076a7d3, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.206893768Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5fb374a2-2d09-404e-b2d8-6f14e3884eab, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.226449607Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 5fb374a2-2d09-404e-b2d8-6f14e3884eab, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.226492068Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.24576494Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.245818912Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.246004546Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.24615344Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.341989588Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:254"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.423980536Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:254"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.424278793Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.424339624Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.424383025Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.424447137Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.424481338Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.424489668Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.424508538Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.424521959Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.628975477Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:254"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.629549211Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.629604282Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.629644643Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.629684784Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.643693878Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:254"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.644117048Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.644144069Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.645461721Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:254"} {"msg":"Configuring new Omicron zone: oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.646540277Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.65238032Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.657399703Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.662361565Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.667104831Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.671993981Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.677161658Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.682059627Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.80298014Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.807805028Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.812120414Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.816492221Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.82052807Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.824875416Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.829724445Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.834298527Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Configuring new Omicron zone: oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.839459843Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.844072796Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.848943366Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.85400484Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.859617067Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.86461245Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.870722699Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.876551562Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.882535418Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.888777642Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.895409604Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.901728889Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:01.954400569Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:271","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:02.011371532Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:271","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:02.044317841Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:271","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:02.07713586Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:271","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:02.109178789Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:271","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:02.112022911Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:296"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:02.112035005Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:188","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:02.1710418Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte1\", ip: 172.30.2.7, mac: MacAddr6([168, 64, 37, 255, 168, 94]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte1\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:02.17110858Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1271","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:02.173882282Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:296"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:02.173902242Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:188","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:02.232848544Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte2\", ip: 172.30.1.5, mac: MacAddr6([168, 64, 37, 255, 224, 81]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 }, vnic: \"vopte2\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:02.232921699Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1271","zone_type":"external_dns"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:02.235042343Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:296"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:02.235057485Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:188","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:02.287170459Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte3\", ip: 172.30.1.6, mac: MacAddr6([168, 64, 37, 255, 241, 146]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 }, vnic: \"vopte3\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:02.287231744Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1271","zone_type":"external_dns"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:02.289209425Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:296"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:02.289229056Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:188","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:02.344338033Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte4\", ip: 172.30.2.5, mac: MacAddr6([168, 64, 37, 255, 195, 22]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte4\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:02.344406726Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1271","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:02.346673379Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:296"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:02.346693415Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:188","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:02.409704704Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte5\", ip: 172.30.2.6, mac: MacAddr6([168, 64, 37, 255, 245, 46]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte5\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:02.409788359Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1271","zone_type":"nexus"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.1.168,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:02.411926131Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:98"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.1.168,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:02.411940044Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:105"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.1.164,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:02.412187576Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:98"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.1.164,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:02.412200373Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:105"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.1.166,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:02.412337963Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:98"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.1.166,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:02.412344301Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:105"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.1.167,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:02.412410124Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:98"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.1.167,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:02.412423644Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:105"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.1.165,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:02.412520274Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:98"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.1.165,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:02.412526959Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:105"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.1.168,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:02.41671543Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:133"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:02.419997223Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.1.164,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:02.420035774Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:133"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.1.165,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:02.421679753Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:133"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.1.166,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:02.423762813Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:133"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:02.423782042Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:02.423790254Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.1.167,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:02.425828004Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:133"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:02.42770634Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:02.427717807Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Configuring new Omicron zone: oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:02.560152007Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:02.567191062Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:02.573861802Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:02.580196096Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:02.587987308Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:02.594350547Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:02.600536801Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:02.605923873Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:02.611336068Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:02.616132896Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:04.212880748Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:48901"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:04.214355075Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"36dc7712-2388-4bc6-9653-e81df185cf6c","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:48901","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":1369,"response_code":"200"} {"msg":"Profile for oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:06.76913254Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:06.769905698Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:06.769919507Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:06.771428013Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:06.771628371Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:06.783145694Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:06.792075389Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:06.800546888Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:06.809137148Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:06.809368669Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005\") failed: Property not found. retry in 73.784561ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:12.25872425Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931\") failed: Property not found. retry in 43.782676ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:12.27163979Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab\") failed: Property not found. retry in 57.021365ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:12.282066396Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb\") failed: Property not found. retry in 54.666944ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:12.291614008Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8\") failed: Property not found. retry in 68.655566ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:12.301272715Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931\") failed: Property not found. retry in 55.943903ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:12.350211086Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab\") failed: Property not found. retry in 117.238609ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:12.361691766Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb\") failed: Property not found. retry in 73.843199ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:12.379757796Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005\") failed: Property not found. retry in 112.79966ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:12.392934858Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8\") failed: Property not found. retry in 57.890285ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:12.415784909Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931\") failed: Property not found. retry in 151.85364ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:12.430232159Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb\") failed: Property not found. retry in 171.394973ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:12.46756748Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8\") failed: Property not found. retry in 220.381116ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:12.499384258Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab\") failed: Property not found. retry in 295.460799ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:12.528958374Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005\") failed: Property not found. retry in 293.181039ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:12.546881061Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931\") failed: Property not found. retry in 440.854479ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:12.595391572Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb\") failed: Property not found. retry in 216.393804ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:12.653615597Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8\") failed: Property not found. retry in 336.108825ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:12.733469769Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab\") failed: Property not found. retry in 438.302988ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:12.838146967Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005\") failed: Property not found. retry in 491.4383ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:12.85453875Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb\") failed: Property not found. retry in 814.647543ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:12.883040752Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931\") failed: Property not found. retry in 590.144852ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:13.0511184Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8\") failed: Property not found. retry in 577.466443ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:13.082743179Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab\") failed: Property not found. retry in 626.623937ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:13.325658632Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005\") failed: Property not found. retry in 1.177061704s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:13.373145582Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931\") failed: Property not found. retry in 1.406304137s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:13.655105667Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8\") failed: Property not found. retry in 995.592628ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:13.674234684Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb\") failed: Property not found. retry in 893.367441ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:13.714458621Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab\") failed: Property not found. retry in 662.43692ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:13.969745922Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005\") failed: Property not found. retry in 1.362729721s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:14.565123138Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb\") failed: Property not found. retry in 854.600882ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:14.620312764Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab\") failed: Property not found. retry in 1.212565791s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:14.642973895Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8\") failed: Property not found. retry in 1.218916561s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:14.684731191Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:14.821413862Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:14.821914301Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931\") failed: Property not found. retry in 650.462742ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:15.079580716Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f\") failed: Property not found. retry in 54.052402ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:15.094197243Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f\") failed: Property not found. retry in 101.221871ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:15.15890234Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f\") failed: Property not found. retry in 250.351817ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:15.272357265Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:15.302712765Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:15.302950976Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb\") failed: Property not found. retry in 1.012181291s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:15.491205635Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f\") failed: Property not found. retry in 355.017252ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:15.538794065Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280\") failed: Property not found. retry in 61.607923ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:15.579089673Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:15.592746045Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:15.593095422Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280\") failed: Property not found. retry in 51.765309ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:15.653941581Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280\") failed: Property not found. retry in 162.580131ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:15.719448981Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931\") failed: Property not found. retry in 970.598703ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:15.743817045Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab\") failed: Property not found. retry in 589.467117ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:15.873977034Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89\") failed: Property not found. retry in 26.205069ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:15.890021473Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280\") failed: Property not found. retry in 408.834507ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:15.904225422Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f\") failed: Property not found. retry in 797.357501ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:15.915294588Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8\") failed: Property not found. retry in 987.603814ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:15.925878793Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89\") failed: Property not found. retry in 78.821697ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:15.93339098Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005\") failed: Property not found. retry in 830.131275ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:15.943770145Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89\") failed: Property not found. retry in 174.935353ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:16.027297726Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89\") failed: Property not found. retry in 413.979757ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:16.215258993Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:16.262112674Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:16.262409888Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:16.262427411Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:16.268961589Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280\") failed: Property not found. retry in 596.708219ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:16.327071729Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:16.360384555Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:16.360695906Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:16.422750651Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:16.42327754Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab\") failed: Property not found. retry in 1.461899552s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:16.476872644Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84\") failed: Property not found. retry in 39.179709ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:16.488366084Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb\") failed: Property not found. retry in 1.01822394s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:16.51731695Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea\") failed: Property not found. retry in 42.682556ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:16.526282364Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84\") failed: Property not found. retry in 111.656775ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:16.540368242Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:16.555133337Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:16.555598293Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea\") failed: Property not found. retry in 86.671507ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:16.580822354Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47\") failed: Property not found. retry in 66.583391ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:16.590907767Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89\") failed: Property not found. retry in 1.042800867s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:16.642622138Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84\") failed: Property not found. retry in 252.117452ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:16.662983889Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47\") failed: Property not found. retry in 54.792996ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:16.67169653Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3\") failed: Property not found. retry in 33.261788ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:16.679947696Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea\") failed: Property not found. retry in 201.104481ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:16.688913838Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:16.690037847Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:16.690458318Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3\") failed: Property not found. retry in 66.651514ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:16.725327366Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931\") failed: Property not found. retry in 1.309885938s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:16.737720564Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f\") failed: Property not found. retry in 1.071746041s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:16.747983285Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47\") failed: Property not found. retry in 144.76741ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:16.75723774Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005\") failed: Property not found. retry in 1.474328338s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:16.787642432Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3\") failed: Property not found. retry in 109.82542ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:16.802057174Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8\") failed: Property not found. retry in 54.114509ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:16.811175291Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8\") failed: Property not found. retry in 54.12668ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:16.875939063Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea\") failed: Property not found. retry in 496.893798ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:16.903684587Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47\") failed: Property not found. retry in 519.806334ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:16.915282314Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3\") failed: Property not found. retry in 400.524852ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:16.932229135Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8\") failed: Property not found. retry in 1.329759375s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:16.942107336Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84\") failed: Property not found. retry in 232.45725ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:16.954800287Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8\") failed: Property not found. retry in 190.980182ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:16.96317927Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280\") failed: Property not found. retry in 1.390906608s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:16.973974955Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8\") failed: Property not found. retry in 48.546722ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:16.983057117Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8\") failed: Property not found. retry in 82.401762ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:17.04211333Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8\") failed: Property not found. retry in 218.635014ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:17.1361424Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8\") failed: Property not found. retry in 514.584687ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:17.167658518Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84\") failed: Property not found. retry in 972.482808ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:17.20118874Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3\") failed: Property not found. retry in 769.842717ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:17.346600897Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8\") failed: Property not found. retry in 423.87545ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:17.368155276Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea\") failed: Property not found. retry in 984.559347ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:17.413840747Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47\") failed: Property not found. retry in 611.336045ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:17.44921832Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb\") failed: Property not found. retry in 1.283713137s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:17.550738931Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:17.619801813Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:17.620210599Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8\") failed: Property not found. retry in 979.859754ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:17.696803624Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89\") failed: Property not found. retry in 660.203843ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:17.709202203Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8\") failed: Property not found. retry in 1.198871491s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:17.805907527Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9\") failed: Property not found. retry in 70.177496ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:17.829163178Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f\") failed: Property not found. retry in 657.985746ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:17.843836811Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9\") failed: Property not found. retry in 144.605659ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:17.910700918Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab\") failed: Property not found. retry in 850.486894ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:17.958554511Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931\") failed: Property not found. retry in 1.30781919s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:18.063773978Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47\") failed: Property not found. retry in 1.404391341s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:18.076055829Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9\") failed: Property not found. retry in 282.544262ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:18.087895767Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3\") failed: Property not found. retry in 1.161333435s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:18.131252107Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84\") failed: Property not found. retry in 875.630185ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:18.189582526Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005\") failed: Property not found. retry in 655.607193ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:18.279455005Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8\") failed: Property not found. retry in 827.381868ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:18.290792354Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280\") failed: Property not found. retry in 681.115662ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:18.381779862Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89\") failed: Property not found. retry in 795.749857ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:18.394723643Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9\") failed: Property not found. retry in 313.735391ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:18.405685162Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea\") failed: Property not found. retry in 625.288697ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:18.418186714Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:18.487883856Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:18.49363193Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:18.498758871Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f\") failed: Property not found. retry in 1.241360926s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:18.51574655Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8\") failed: Property not found. retry in 673.956165ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:19.163275541Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8\") failed: Property not found. retry in 1.384623276s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:19.227220837Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea\") failed: Property not found. retry in 1.164569336s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:19.313966813Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab\") failed: Property not found. retry in 1.223341897s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:19.345126637Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84\") failed: Property not found. retry in 655.297861ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:19.36556763Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb\") failed: Property not found. retry in 801.891772ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:19.390605019Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89\") failed: Property not found. retry in 1.075101038s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:19.422275571Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3\") failed: Property not found. retry in 931.298588ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:19.492774648Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005\") failed: Property not found. retry in 802.264235ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:19.518085075Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9\") failed: Property not found. retry in 743.657871ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:19.532440651Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8\") failed: Property not found. retry in 883.80214ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:19.549430965Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280\") failed: Property not found. retry in 1.045305738s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:19.602090935Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897\") failed: Property not found. retry in 26.486186ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:19.627887264Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3\") failed: Property not found. retry in 27.564147ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:19.726284588Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d\") failed: Property not found. retry in 41.974982ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:19.751460854Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931\") failed: Property not found. retry in 965.057332ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:19.767841848Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47\") failed: Property not found. retry in 1.19189144s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:19.796552192Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f\") failed: Property not found. retry in 682.625165ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:19.809162656Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897\") failed: Property not found. retry in 87.61416ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:19.820857479Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3\") failed: Property not found. retry in 146.618338ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:19.831663271Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d\") failed: Property not found. retry in 81.476365ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:19.842227232Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8\") failed: Property not found. retry in 1.402018895s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:19.86777726Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897\") failed: Property not found. retry in 123.55185ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:19.92200972Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d\") failed: Property not found. retry in 150.331112ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:19.937782628Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3\") failed: Property not found. retry in 156.321316ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:19.991685087Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84\") failed: Property not found. retry in 1.274962283s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:20.036965971Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897\") failed: Property not found. retry in 425.884989ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:20.059087108Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d\") failed: Property not found. retry in 207.861678ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:20.100413354Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3\") failed: Property not found. retry in 420.423249ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:20.160107183Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb\") failed: Property not found. retry in 1.005621832s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:20.207672545Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9\") failed: Property not found. retry in 1.006858778s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:20.291815625Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d\") failed: Property not found. retry in 453.705014ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:20.321064149Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005\") failed: Property not found. retry in 1.116648952s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:20.333905975Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3\") failed: Property not found. retry in 575.540097ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:20.441220963Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8\") failed: Property not found. retry in 1.47764618s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:20.454977562Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea\") failed: Property not found. retry in 755.876666ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:20.49407946Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897\") failed: Property not found. retry in 750.210275ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:20.504225918Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89\") failed: Property not found. retry in 500.31816ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:20.515059501Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f\") failed: Property not found. retry in 1.435231184s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:20.525850115Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab\") failed: Property not found. retry in 661.017687ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:20.58485957Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3\") failed: Property not found. retry in 646.735633ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:20.594509408Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8\") failed: Property not found. retry in 1.484217902s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:20.627553187Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280\") failed: Property not found. retry in 584.8619ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:20.662243363Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931\") failed: Property not found. retry in 548.370746ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:20.748553355Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d\") failed: Property not found. retry in 710.185778ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:20.786824245Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47\") failed: Property not found. retry in 889.028556ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:21.0030282Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89\") failed: Property not found. retry in 807.850881ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:21.026935161Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3\") failed: Property not found. retry in 535.628831ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:21.03686308Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb\") failed: Property not found. retry in 561.102863ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:21.225563336Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3\") failed: Property not found. retry in 1.251957016s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:21.25238521Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea\") failed: Property not found. retry in 726.124947ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:21.26002844Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab\") failed: Property not found. retry in 644.480277ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:21.267077364Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280\") failed: Property not found. retry in 887.586027ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:21.274678356Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8\") failed: Property not found. retry in 1.421386643s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:21.281844862Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897\") failed: Property not found. retry in 1.374254511s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:21.2898328Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931\") failed: Property not found. retry in 1.401477448s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:21.306238421Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9\") failed: Property not found. retry in 674.355956ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:21.312856455Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84\") failed: Property not found. retry in 1.180521292s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:21.323533252Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005\") failed: Property not found. retry in 1.090599765s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:21.463593999Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d\") failed: Property not found. retry in 533.836509ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:21.509164329Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3\") failed: Property not found. retry in 621.809628ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:21.582387342Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb\") failed: Property not found. retry in 1.076643625s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:21.797197259Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89\") failed: Property not found. retry in 1.468152371s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:21.846223131Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47\") failed: Property not found. retry in 645.610372ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:21.903671966Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab\") failed: Property not found. retry in 1.294464112s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:21.922719818Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8\") failed: Property not found. retry in 511.616591ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:21.946012814Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f\") failed: Property not found. retry in 1.225020481s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:21.974313403Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea\") failed: Property not found. retry in 1.135990237s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:21.996463268Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9\") failed: Property not found. retry in 871.892873ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:22.005086551Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d\") failed: Property not found. retry in 1.175945437s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:22.053647727Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8\") failed: Property not found. retry in 722.156794ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:22.124442349Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280\") failed: Property not found. retry in 815.81251ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:22.173494817Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3\") failed: Property not found. retry in 667.70455ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:22.213752279Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8\") failed: Property not found. retry in 1.322894095s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:22.468386048Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84\") failed: Property not found. retry in 1.303783447s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:22.515967131Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3\") failed: Property not found. retry in 1.233586401s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:22.523197921Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47\") failed: Property not found. retry in 1.387017966s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:22.559801474Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005\") failed: Property not found. retry in 880.793486ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:22.567621631Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897\") failed: Property not found. retry in 1.269317612s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:22.678596733Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8\") failed: Property not found. retry in 1.021545349s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:22.716202937Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931\") failed: Property not found. retry in 1.180331101s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:22.724536681Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8\") failed: Property not found. retry in 1.097215205s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:22.857238009Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb\") failed: Property not found. retry in 1.416632442s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:22.886054738Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3\") failed: Property not found. retry in 976.229938ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:22.893783977Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9\") failed: Property not found. retry in 653.95197ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:22.906020068Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280\") failed: Property not found. retry in 1.16243798s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:23.002068504Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea\") failed: Property not found. retry in 784.89511ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:23.146852148Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f\") failed: Property not found. retry in 1.229085348s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:23.213826359Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab\") failed: Property not found. retry in 1.000905646s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:23.233093175Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d\") failed: Property not found. retry in 543.246761ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:23.2452636Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89\") failed: Property not found. retry in 1.294604662s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:23.331030747Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005\") failed: Property not found. retry in 1.031599522s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:23.461756779Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9\") failed: Property not found. retry in 854.70982ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:23.574419731Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8\") failed: Property not found. retry in 911.941395ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:23.777115511Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3\") failed: Property not found. retry in 782.62244ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:23.789952741Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d\") failed: Property not found. retry in 544.132105ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:23.803538544Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8\") failed: Property not found. retry in 626.849359ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:23.814738727Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84\") failed: Property not found. retry in 1.004281445s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:23.835048294Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3\") failed: Property not found. retry in 669.573774ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:23.885826327Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931\") failed: Property not found. retry in 1.371594437s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:23.920220134Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea\") failed: Property not found. retry in 932.488253ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:23.947978387Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47\") failed: Property not found. retry in 962.306825ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:23.962141049Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8\") failed: Property not found. retry in 837.634325ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:23.972722887Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897\") failed: Property not found. retry in 929.518849ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:23.982138315Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280\") failed: Property not found. retry in 737.197859ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:24.179912398Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab\") failed: Property not found. retry in 1.453781968s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:24.249286894Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb\") failed: Property not found. retry in 812.620204ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:24.317815427Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d\") failed: Property not found. retry in 724.578447ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:24.36303981Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9\") failed: Property not found. retry in 547.162866ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:24.443980261Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8\") failed: Property not found. retry in 1.45855576s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:24.455519437Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f\") failed: Property not found. retry in 720.043853ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:24.465952635Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005\") failed: Property not found. retry in 1.260229594s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:24.509096728Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3\") failed: Property not found. retry in 908.229042ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:24.566787379Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3\") failed: Property not found. retry in 1.276838215s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:24.586151454Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89\") failed: Property not found. retry in 637.706408ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:24.640429867Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8\") failed: Property not found. retry in 540.389734ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:24.70183244Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8\") failed: Property not found. retry in 526.483659ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:24.823828721Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84\") failed: Property not found. retry in 981.275578ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:24.853294029Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea\") failed: Property not found. retry in 535.336081ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:24.907785612Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:24.910761532Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:24.911494499Z","hostname":"buskin","pid":1671,"zone":"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897\") failed: Property not found. retry in 824.026143ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:24.929315957Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47\") failed: Property not found. retry in 1.03133594s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:24.939349178Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280\") failed: Property not found. retry in 740.678357ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:24.949303937Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9\") failed: Property not found. retry in 1.153701251s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:25.004601051Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d\") failed: Property not found. retry in 1.365165821s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:25.10095557Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749\") failed: Property not found. retry in 41.832486ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:25.111454297Z","hostname":"buskin","pid":1671,"zone":"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb\") failed: Property not found. retry in 624.913359ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:25.142152514Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749\") failed: Property not found. retry in 72.90779ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:25.162276422Z","hostname":"buskin","pid":1671,"zone":"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f\") failed: Property not found. retry in 687.898213ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:25.197220996Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749\") failed: Property not found. retry in 160.733176ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:25.247821516Z","hostname":"buskin","pid":1671,"zone":"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8\") failed: Property not found. retry in 858.361032ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:25.258674949Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89\") failed: Property not found. retry in 1.18166427s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:25.290801642Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931\") failed: Property not found. retry in 835.680853ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:25.305799267Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8\") failed: Property not found. retry in 883.241585ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:25.363490963Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749\") failed: Property not found. retry in 290.410849ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:25.423149526Z","hostname":"buskin","pid":1671,"zone":"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea\") failed: Property not found. retry in 1.154509756s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:25.454762256Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3\") failed: Property not found. retry in 820.278959ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:25.486178947Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280\") failed: Property not found. retry in 516.708286ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:25.701237143Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab\") failed: Property not found. retry in 1.440033695s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:25.717064086Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749\") failed: Property not found. retry in 1.129457105s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:25.725118651Z","hostname":"buskin","pid":1671,"zone":"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897\") failed: Property not found. retry in 1.293932411s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:25.765343723Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb\") failed: Property not found. retry in 1.400175554s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:25.779019261Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005\") failed: Property not found. retry in 1.120054671s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:25.788640144Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84\") failed: Property not found. retry in 732.120129ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:25.846938462Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3\") failed: Property not found. retry in 1.225585124s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:25.875344022Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f\") failed: Property not found. retry in 1.112935568s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:25.897820054Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8\") failed: Property not found. retry in 1.355447849s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:25.92832655Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47\") failed: Property not found. retry in 880.318719ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:25.983071705Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8\") failed: Property not found. retry in 1.467072936s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:26.129305087Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931\") failed: Property not found. retry in 884.497082ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:26.154313162Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9\") failed: Property not found. retry in 575.621586ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:26.172047473Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280\") failed: Property not found. retry in 627.370601ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:26.229199442Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8\") failed: Property not found. retry in 566.484187ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:26.258968481Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3\") failed: Property not found. retry in 1.369362551s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:26.319220312Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d\") failed: Property not found. retry in 770.975219ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:26.480232158Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89\") failed: Property not found. retry in 619.070897ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:26.488972583Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84\") failed: Property not found. retry in 533.852154ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:26.59102992Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea\") failed: Property not found. retry in 1.274669641s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:26.622576619Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9\") failed: Property not found. retry in 977.601561ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:26.759611293Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8\") failed: Property not found. retry in 507.594086ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:26.836847687Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749\") failed: Property not found. retry in 1.335342673s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:26.867869286Z","hostname":"buskin","pid":1671,"zone":"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280\") failed: Property not found. retry in 1.499040093s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:26.875854775Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47\") failed: Property not found. retry in 889.15725ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:26.883361883Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005\") failed: Property not found. retry in 1.066452782s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:26.920462531Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f\") failed: Property not found. retry in 757.729812ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:27.024204687Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931\") failed: Property not found. retry in 691.587817ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:27.050915706Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897\") failed: Property not found. retry in 1.037954197s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:27.071172941Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3\") failed: Property not found. retry in 1.196853071s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:27.113839925Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89\") failed: Property not found. retry in 1.407198485s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:27.12172617Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84\") failed: Property not found. retry in 997.990736ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:27.13722879Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab\") failed: Property not found. retry in 1.01477671s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:27.170382797Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb\") failed: Property not found. retry in 544.938008ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:27.192243744Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d\") failed: Property not found. retry in 528.092188ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:27.264245022Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8\") failed: Property not found. retry in 656.582275ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:27.297481756Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8\") failed: Property not found. retry in 604.979804ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:27.357812278Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8\") failed: Property not found. retry in 914.49345ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:27.610419737Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3\") failed: Property not found. retry in 1.406795469s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:27.703901239Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb\") failed: Property not found. retry in 1.159776619s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:27.749069074Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931\") failed: Property not found. retry in 645.116338ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:27.760641828Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9\") failed: Property not found. retry in 1.329554972s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:27.772025975Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47\") failed: Property not found. retry in 828.879533ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:27.785785787Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f\") failed: Property not found. retry in 1.181582232s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:27.797879778Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d\") failed: Property not found. retry in 723.052462ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:27.807494032Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea\") failed: Property not found. retry in 834.024078ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:27.913378935Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8\") failed: Property not found. retry in 746.543889ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:27.967667356Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8\") failed: Property not found. retry in 1.360519034s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:27.980957555Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005\") failed: Property not found. retry in 1.287471169s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:28.002091227Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897\") failed: Property not found. retry in 1.01554074s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:28.124434438Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84\") failed: Property not found. retry in 1.457395918s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:28.148115816Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab\") failed: Property not found. retry in 1.330032818s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:28.197947315Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749\") failed: Property not found. retry in 1.359044336s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:28.217156524Z","hostname":"buskin","pid":1671,"zone":"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3\") failed: Property not found. retry in 1.494957237s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:28.324822571Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280\") failed: Property not found. retry in 989.560464ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:28.390575168Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931\") failed: Property not found. retry in 1.410940451s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:28.418836128Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8\") failed: Property not found. retry in 1.179194246s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:28.538483044Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d\") failed: Property not found. retry in 930.275003ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:28.547195242Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89\") failed: Property not found. retry in 1.245981178s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:28.555807165Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47\") failed: Property not found. retry in 1.337780075s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:28.628522503Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8\") failed: Property not found. retry in 723.3253ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:28.735421699Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea\") failed: Property not found. retry in 1.425452431s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:28.761110549Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb\") failed: Property not found. retry in 694.971853ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:28.922575219Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f\") failed: Property not found. retry in 993.967285ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:28.992355746Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9\") failed: Property not found. retry in 1.182794533s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:29.115953718Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3\") failed: Property not found. retry in 821.176063ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:29.126863163Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897\") failed: Property not found. retry in 669.195227ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:29.152266116Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005\") failed: Property not found. retry in 911.799214ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:29.30317146Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8\") failed: Property not found. retry in 1.375464717s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:29.356800652Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280\") failed: Property not found. retry in 1.231507828s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:29.395253405Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8\") failed: Property not found. retry in 735.455446ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:29.473514443Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d\") failed: Property not found. retry in 1.050316201s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:29.491078135Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab\") failed: Property not found. retry in 1.340006949s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:29.542092907Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749\") failed: Property not found. retry in 944.397337ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:29.591742579Z","hostname":"buskin","pid":1671,"zone":"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84\") failed: Property not found. retry in 1.320564722s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:29.620810116Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb\") failed: Property not found. retry in 977.68229ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:29.629834085Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8\") failed: Property not found. retry in 854.006515ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:29.730614334Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89\") failed: Property not found. retry in 517.902398ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:29.814468424Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897\") failed: Property not found. retry in 515.587291ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:29.834300057Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3\") failed: Property not found. retry in 1.099004438s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:29.845993278Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931\") failed: Property not found. retry in 863.177395ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:29.856930707Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3\") failed: Property not found. retry in 604.35648ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:29.961379053Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47\") failed: Property not found. retry in 706.253413ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:29.980312681Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f\") failed: Property not found. retry in 756.485462ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:29.998975054Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea\") failed: Property not found. retry in 880.021901ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:30.200618574Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8\") failed: Property not found. retry in 1.24346565s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:30.22120181Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005\") failed: Property not found. retry in 1.069298859s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:30.229727002Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9\") failed: Property not found. retry in 544.526657ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:30.31524805Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89\") failed: Property not found. retry in 1.251630119s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:30.343299931Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897\") failed: Property not found. retry in 961.702375ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:30.361530088Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749\") failed: Property not found. retry in 1.37745116s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:30.549678669Z","hostname":"buskin","pid":1671,"zone":"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d\") failed: Property not found. retry in 1.23453117s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:30.557372581Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3\") failed: Property not found. retry in 1.259756912s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:30.577332961Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8\") failed: Property not found. retry in 973.077071ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:30.597559503Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb\") failed: Property not found. retry in 520.003913ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:30.620003516Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280\") failed: Property not found. retry in 1.472757079s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:30.638662975Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47\") failed: Property not found. retry in 683.65169ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:30.699282868Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931\") failed: Property not found. retry in 1.29798235s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:30.735501805Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8\") failed: Property not found. retry in 898.774657ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:30.747814607Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f\") failed: Property not found. retry in 1.323258301s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:30.76903362Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9\") failed: Property not found. retry in 1.104265723s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:30.873208067Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab\") failed: Property not found. retry in 1.374818483s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:30.896681562Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84\") failed: Property not found. retry in 598.078882ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:30.957007615Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3\") failed: Property not found. retry in 1.214570136s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:30.9655346Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea\") failed: Property not found. retry in 855.911421ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:31.097442429Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb\") failed: Property not found. retry in 912.28615ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:31.153610315Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005\") failed: Property not found. retry in 673.240371ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:31.315593713Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897\") failed: Property not found. retry in 782.787448ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:31.338243252Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47\") failed: Property not found. retry in 1.465595907s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:31.399659067Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8\") failed: Property not found. retry in 765.462172ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:31.483614169Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84\") failed: Property not found. retry in 855.23974ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:31.575479404Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8\") failed: Property not found. retry in 1.352513332s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:31.590407656Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89\") failed: Property not found. retry in 1.499870125s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:31.619840956Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8\") failed: Property not found. retry in 1.239510261s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:31.670085966Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d\") failed: Property not found. retry in 1.164199206s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:31.817749746Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3\") failed: Property not found. retry in 544.611764ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:31.860182287Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749\") failed: Property not found. retry in 669.66683ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:31.950871898Z","hostname":"buskin","pid":1671,"zone":"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea\") failed: Property not found. retry in 663.766365ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:31.973507702Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9\") failed: Property not found. retry in 798.970207ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:31.996595658Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005\") failed: Property not found. retry in 830.927793ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:32.012543855Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931\") failed: Property not found. retry in 616.045976ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:32.056611541Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb\") failed: Property not found. retry in 648.119766ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:32.09097354Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f\") failed: Property not found. retry in 655.589111ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:32.118564351Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280\") failed: Property not found. retry in 828.71288ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:32.140028473Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897\") failed: Property not found. retry in 1.191156134s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:32.156225477Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3\") failed: Property not found. retry in 1.380092268s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:32.20228322Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8\") failed: Property not found. retry in 802.116161ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:32.267570028Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab\") failed: Property not found. retry in 823.928189ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:32.294902957Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3\") failed: Property not found. retry in 1.228346562s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:32.424996992Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84\") failed: Property not found. retry in 1.375818071s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:32.449118439Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749\") failed: Property not found. retry in 1.289854834s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:32.641507082Z","hostname":"buskin","pid":1671,"zone":"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea\") failed: Property not found. retry in 1.369099257s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:32.666196086Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931\") failed: Property not found. retry in 1.487979726s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:32.694653866Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb\") failed: Property not found. retry in 628.194202ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:32.759671554Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f\") failed: Property not found. retry in 1.224585881s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:32.792801606Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9\") failed: Property not found. retry in 596.921925ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:32.815176669Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005\") failed: Property not found. retry in 620.554643ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:32.864680837Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47\") failed: Property not found. retry in 635.976146ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:32.884438452Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8\") failed: Property not found. retry in 698.024191ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:32.930687226Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8\") failed: Property not found. retry in 936.193147ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:32.96451092Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280\") failed: Property not found. retry in 926.777435ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:32.98846423Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d\") failed: Property not found. retry in 772.452198ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:33.000154661Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8\") failed: Property not found. retry in 580.588001ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:33.091095157Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab\") failed: Property not found. retry in 730.338612ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:33.132794484Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89\") failed: Property not found. retry in 909.153621ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:33.144411931Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897\") failed: Property not found. retry in 908.414201ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:33.364511215Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb\") failed: Property not found. retry in 1.245700256s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:33.402561036Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9\") failed: Property not found. retry in 594.404872ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:33.425770924Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005\") failed: Property not found. retry in 1.110759314s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:33.507602752Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47\") failed: Property not found. retry in 1.31054936s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:33.534767201Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3\") failed: Property not found. retry in 1.362944973s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:33.595918767Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8\") failed: Property not found. retry in 1.408827627s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:33.643087124Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3\") failed: Property not found. retry in 1.268858699s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:33.667021911Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8\") failed: Property not found. retry in 663.423181ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:33.685720793Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d\") failed: Property not found. retry in 1.321431788s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:33.786127306Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84\") failed: Property not found. retry in 614.964211ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:33.840500752Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab\") failed: Property not found. retry in 1.26827785s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:33.874523858Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8\") failed: Property not found. retry in 569.18357ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:33.912803811Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280\") failed: Property not found. retry in 1.485402299s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:33.926787685Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749\") failed: Property not found. retry in 779.235401ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:33.944292739Z","hostname":"buskin","pid":1671,"zone":"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f\") failed: Property not found. retry in 541.559476ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:34.030130238Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9\") failed: Property not found. retry in 565.259603ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:34.038063612Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea\") failed: Property not found. retry in 897.117155ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:34.047721729Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89\") failed: Property not found. retry in 751.916798ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:34.064882291Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931\") failed: Property not found. retry in 668.018376ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:34.196131933Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897\") failed: Property not found. retry in 579.829951ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:34.285841343Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8\") failed: Property not found. retry in 1.189080781s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:34.360815635Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84\") failed: Property not found. retry in 688.679067ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:34.466898404Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8\") failed: Property not found. retry in 1.203628075s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:34.493762144Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:34.505575698Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:42414"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:34.507241331Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"734349c5-247e-4e09-b76a-bba0740e3efc","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:42414","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":1098,"response_code":"200"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f\") failed: Property not found. retry in 967.615655ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:34.585176519Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9\") failed: Property not found. retry in 516.972618ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:34.615452495Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005\") failed: Property not found. retry in 936.3656ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:34.632083074Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb\") failed: Property not found. retry in 1.397271796s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:34.660301583Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749\") failed: Property not found. retry in 577.384959ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:34.734378421Z","hostname":"buskin","pid":1671,"zone":"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89\") failed: Property not found. retry in 926.751213ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:34.827508574Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47\") failed: Property not found. retry in 1.161501561s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:34.857294493Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931\") failed: Property not found. retry in 1.186548117s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:34.875859768Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897\") failed: Property not found. retry in 1.042963304s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:34.884056395Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3\") failed: Property not found. retry in 1.077288478s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:34.948003334Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea\") failed: Property not found. retry in 986.478499ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:34.955630839Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3\") failed: Property not found. retry in 813.528998ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:34.970909457Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8\") failed: Property not found. retry in 696.464815ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:35.064870384Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d\") failed: Property not found. retry in 695.836745ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:35.120863781Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9\") failed: Property not found. retry in 963.334347ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:35.143899951Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab\") failed: Property not found. retry in 605.419066ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:35.155964121Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84\") failed: Property not found. retry in 1.175255178s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:35.167043026Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749\") failed: Property not found. retry in 1.494233737s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:35.323464596Z","hostname":"buskin","pid":1671,"zone":"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280\") failed: Property not found. retry in 1.090235597s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:35.426993278Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8\") failed: Property not found. retry in 567.177448ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:35.562882653Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f\") failed: Property not found. retry in 649.255035ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:35.571672415Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005\") failed: Property not found. retry in 1.161215132s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:35.579713133Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8\") failed: Property not found. retry in 1.021362907s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:35.709268081Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89\") failed: Property not found. retry in 1.186915793s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:35.765930129Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8\") failed: Property not found. retry in 680.1217ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:35.77392877Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab\") failed: Property not found. retry in 586.575866ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:35.782316936Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3\") failed: Property not found. retry in 993.411719ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:35.796053768Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d\") failed: Property not found. retry in 1.320203859s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:35.827533347Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897\") failed: Property not found. retry in 695.866408ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:35.941942567Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea\") failed: Property not found. retry in 708.763674ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:35.955388662Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47\") failed: Property not found. retry in 1.089601462s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:36.031495001Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3\") failed: Property not found. retry in 608.174741ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:36.040227355Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb\") failed: Property not found. retry in 712.368153ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:36.071077331Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931\") failed: Property not found. retry in 1.056519508s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:36.07961546Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9\") failed: Property not found. retry in 975.307444ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:36.118521443Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8\") failed: Property not found. retry in 1.460304557s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:36.141702281Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f\") failed: Property not found. retry in 1.242133468s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:36.232565757Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84\") failed: Property not found. retry in 780.164852ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:36.354435842Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab\") failed: Property not found. retry in 1.154460622s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:36.379590738Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8\") failed: Property not found. retry in 1.111689251s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:36.465429081Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280\") failed: Property not found. retry in 662.50011ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:36.527914392Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897\") failed: Property not found. retry in 1.257001208s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:36.648220879Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3\") failed: Property not found. retry in 1.345960872s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:36.65846373Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea\") failed: Property not found. retry in 516.618552ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:36.673424636Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8\") failed: Property not found. retry in 507.222555ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:36.740786966Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005\") failed: Property not found. retry in 755.308081ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:36.7519814Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb\") failed: Property not found. retry in 1.093171422s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:36.791703963Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3\") failed: Property not found. retry in 611.095106ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:36.799447533Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749\") failed: Property not found. retry in 1.228491433s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:36.828411233Z","hostname":"buskin","pid":1671,"zone":"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89\") failed: Property not found. retry in 735.606387ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:36.963226835Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9\") failed: Property not found. retry in 621.829941ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:37.104472645Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47\") failed: Property not found. retry in 622.733035ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:37.130539234Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84\") failed: Property not found. retry in 556.792069ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:37.142286481Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931\") failed: Property not found. retry in 779.098995ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:37.14998793Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d\") failed: Property not found. retry in 1.174344628s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:37.157912704Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea\") failed: Property not found. retry in 862.308404ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:37.20017492Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280\") failed: Property not found. retry in 1.274199295s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:37.208859862Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8\") failed: Property not found. retry in 870.051789ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:37.257588146Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3\") failed: Property not found. retry in 855.494296ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:37.419354618Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f\") failed: Property not found. retry in 1.3010175s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:37.484842222Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005\") failed: Property not found. retry in 1.140703332s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:37.516486677Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab\") failed: Property not found. retry in 1.240765203s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:37.543293864Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8\") failed: Property not found. retry in 836.292924ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:37.585964139Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8\") failed: Property not found. retry in 1.083215638s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:37.610400398Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84\") failed: Property not found. retry in 1.485345347s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:37.709132346Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89\") failed: Property not found. retry in 527.822211ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:37.718466035Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9\") failed: Property not found. retry in 508.19184ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:37.735740158Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47\") failed: Property not found. retry in 899.252333ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:37.761704864Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb\") failed: Property not found. retry in 1.485777389s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:37.894560378Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897\") failed: Property not found. retry in 723.470783ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:37.914148908Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931\") failed: Property not found. retry in 1.224981988s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:37.93948506Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3\") failed: Property not found. retry in 1.225095941s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:38.015211724Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749\") failed: Property not found. retry in 1.176729875s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:38.065946387Z","hostname":"buskin","pid":1671,"zone":"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea\") failed: Property not found. retry in 937.696255ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:38.072912487Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8\") failed: Property not found. retry in 623.011774ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:38.137203252Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9\") failed: Property not found. retry in 796.26675ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:38.254871165Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89\") failed: Property not found. retry in 1.321465245s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:38.263677871Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3\") failed: Property not found. retry in 713.863741ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:38.284144732Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d\") failed: Property not found. retry in 1.291300595s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:38.341496697Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8\") failed: Property not found. retry in 635.900188ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:38.430664531Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280\") failed: Property not found. retry in 1.07767359s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:38.492737011Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897\") failed: Property not found. retry in 1.096236202s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:38.647381019Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005\") failed: Property not found. retry in 1.129476604s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:38.666811775Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47\") failed: Property not found. retry in 748.58665ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:38.674489693Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8\") failed: Property not found. retry in 593.827618ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:38.703556345Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8\") failed: Property not found. retry in 945.413097ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:38.770062864Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab\") failed: Property not found. retry in 719.696634ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:38.793259423Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f\") failed: Property not found. retry in 676.830842ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:38.801116635Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3\") failed: Property not found. retry in 1.061603308s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:39.008331171Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea\") failed: Property not found. retry in 541.285616ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:39.019003182Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9\") failed: Property not found. retry in 1.113758117s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:39.05932234Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47\") failed: Property not found. retry in 1.11250987s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:39.448718969Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749\") failed: Property not found. retry in 566.254566ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:39.496477568Z","hostname":"buskin","pid":1671,"zone":"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897\") failed: Property not found. retry in 781.896592ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:39.979615852Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749\") failed: Property not found. retry in 663.641197ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:40.190919368Z","hostname":"buskin","pid":1671,"zone":"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring address fd00:1122:3344:101::c exists","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:40.388847207Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2002"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::c, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:40.388900718Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"Ensuring address fd00:1122:3344:101::c exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:40.960317565Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2010"} {"msg":"Zone using sled underlay as gateway","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:40.961663928Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2029"} {"msg":"Setting up Nexus service","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:45.399268387Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2073"} {"msg":"Ensuring address for OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:45.399330548Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:653"} {"msg":"Ensuring address fd00:1122:3344:101::b exists","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:52.019974532Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2002"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::b, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:52.020041503Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"Ensuring address fd00:1122:3344:101::b exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:52.582617243Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2010"} {"msg":"Zone using sled underlay as gateway","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:52.582682345Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2029"} {"msg":"Setting up Nexus service","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:53.188923615Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2073"} {"msg":"Ensuring address for OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:53.188978607Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:653"} {"msg":"Ensuring address fd00:1122:3344:101::a exists","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:59.048672811Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2002"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::a, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:59.048732452Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"Ensuring address fd00:1122:3344:101::a exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:59.6026622Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2010"} {"msg":"Zone using sled underlay as gateway","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:59.602730112Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2029"} {"msg":"failed to notify nexus about a new pool oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:59.624487065Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"170.154563594s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cc74b0a2-490e-4424-adad-bb6a277ade47/zpools/e4b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:17:59.911498596Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"170.443586084s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cc74b0a2-490e-4424-adad-bb6a277ade47/zpools/24b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"Setting up Nexus service","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:59.942962776Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2073"} {"msg":"Ensuring address for OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:17:59.942981097Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:653"} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", sled_id: cc74b0a2-490e-4424-adad-bb6a277ade47, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:18:00.452302982Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"170.985514979s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:18:00.511477282Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"171.043211121s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cc74b0a2-490e-4424-adad-bb6a277ade47/zpools/616b26df-e62a-4c68-b506-f4a923d8aaf7): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_31bd71cd-4736-4a12-a387-9b74b050396f on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:18:00.901646729Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"171.433480852s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cc74b0a2-490e-4424-adad-bb6a277ade47/zpools/31bd71cd-4736-4a12-a387-9b74b050396f): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: cc74b0a2-490e-4424-adad-bb6a277ade47, variant: M2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:18:01.66192179Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"172.195401676s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:18:01.702133457Z","hostname":"buskin","pid":1671,"sled_id":"cc74b0a2-490e-4424-adad-bb6a277ade47","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1199","retry_after":"8.205153698s","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:18:01.981532691Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"172.513716853s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cc74b0a2-490e-4424-adad-bb6a277ade47/zpools/14b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: cc74b0a2-490e-4424-adad-bb6a277ade47, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:18:02.081592502Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"172.614092753s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:18:04.26234968Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"174.793662363s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cc74b0a2-490e-4424-adad-bb6a277ade47/zpools/b462a7f7-b628-40fe-80ff-4e4189e2d62b): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: cc74b0a2-490e-4424-adad-bb6a277ade47, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:18:04.262435092Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"174.797899507s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:04.555879649Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:40704"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:04.556011612Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"d7be1ecf-8369-4982-9bab-672e57952cc6","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:40704","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":54,"response_code":"200"} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: cc74b0a2-490e-4424-adad-bb6a277ade47, variant: M2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:18:04.635045068Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"175.168377632s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:18:05.01803936Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"175.550098442s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cc74b0a2-490e-4424-adad-bb6a277ade47/zpools/14b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:18:05.179964556Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"175.710480042s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/cc74b0a2-490e-4424-adad-bb6a277ade47/zpools/e4b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:05.366647923Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_nexus_264d5dd4-3eb2-4adb-b639-f0a26ab5534d"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:05.36671175Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_crucible_cf8e7366-58a5-4520-b2ad-904e641779d8"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:05.366727406Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_crucible_6c91a708-fa2b-487f-8528-4d94d9438931"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:05.366738121Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_crucible_pantry_282e3323-14c4-4d62-8807-8693fcbf40ea"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:05.366753413Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_crucible_5bbf29d8-d45a-494e-b486-773cde91df47"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:05.366761185Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_crucible_5fb374a2-2d09-404e-b2d8-6f14e3884eab"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:05.366780319Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_crucible_pantry_925e71e3-e0b8-4973-8ff8-5438a755ef84"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:05.36678796Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_crucible_7f367ce2-eb70-48d4-b9f5-2554c178c7d8"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:05.366793408Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_crucible_81f36664-5b2e-47bc-8f86-8336a864a7bb"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:05.366960905Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_external_dns_0a603215-e206-4aa0-8930-250f278a0d89"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:05.366971696Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_crucible_056f8fb6-5500-4b9e-92d0-2be6e076a7d3"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:05.366979619Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_clickhouse_74802856-0b1c-433c-9e82-d4aedb5e0749"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:05.366988262Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_crucible_85a1d6ba-7262-4d27-a9fc-1d1499e79005"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:05.367001959Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_oximeter_a87cb16a-062c-4865-a37d-8e0d7e2bd2b9"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:05.36700989Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_external_dns_3f949516-3930-421a-bae2-b58ff4ea0dc8"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:05.367017307Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_crucible_pantry_ecba71f7-525f-4818-b686-c9ff63ca561f"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:05.367024816Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_crucible_3de24d1d-6129-402e-99c6-cbe1a3727280"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:05.367040585Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_nexus_4ea96463-fca7-4b4a-8a70-4e88cc368897"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:05.367048704Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2828","zone":"oxz_nexus_933c0d3a-d49d-431b-8f65-51222f67e8f3"} {"msg":"Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:05.367634264Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:198"} {"msg":"Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:05.369661263Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:198"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:05.370956325Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"0ba1751a-d703-4c61-a74d-95586a8df970","remote_addr":"[fd00:1122:3344:101::1]:42557","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":64414868,"response_code":"204"} {"msg":"Finished setting up services","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:05.382426856Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1059"} {"msg":"Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:05.383493472Z","hostname":"buskin","pid":1671,"component":"RSS","file":"common/src/ledger.rs:198"} {"msg":"Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:05.383665446Z","hostname":"buskin","pid":1671,"component":"RSS","file":"common/src/ledger.rs:198"} {"msg":"Handing off control to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:05.386978858Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:549"} {"msg":"Nexus address: [fd00:1122:3344:101::c]:12221","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:05.386994128Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:551"} {"msg":"rack_network_config: RackNetworkConfigV1 {\n bgp: [],\n infra_ip_first: 192.168.1.30,\n infra_ip_last: 192.168.1.30,\n ports: [\n PortConfigV1 {\n addresses: [\n V4(\n Ipv4Network {\n addr: 10.151.1.174,\n prefix: 24,\n },\n ),\n ],\n autoneg: false,\n bgp_peers: [],\n port: \"qsfp0\",\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Network {\n addr: 0.0.0.0,\n prefix: 0,\n },\n ),\n nexthop: 10.151.1.100,\n },\n ],\n switch: Switch0,\n uplink_port_fec: None,\n uplink_port_speed: Speed40G,\n },\n ],\n rack_subnet: Ipv6Network {\n addr: fd00:1122:3344:100::,\n prefix: 56,\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:05.425258775Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/5d4f5e32-9bf1-4871-a9fd-8e138cda5d0f/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:05.426435894Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/5d4f5e32-9bf1-4871-a9fd-8e138cda5d0f/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:05.477661648Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/5d4f5e32-9bf1-4871-a9fd-8e138cda5d0f/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:05.640878126Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", sled_id: cc74b0a2-490e-4424-adad-bb6a277ade47, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:18:05.659162264Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"176.192591631s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/5d4f5e32-9bf1-4871-a9fd-8e138cda5d0f/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:05.733935726Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/5d4f5e32-9bf1-4871-a9fd-8e138cda5d0f/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:05.961426208Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ed03abb8-c48f-469b-a9bf-7bf6aa1ad101\", \"content-length\": \"160\", \"date\": \"Fri, 01 Mar 2024 23:18:06 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: ip-pool with name \\\"oxide-service-pool\\\"\", request_id: \"ed03abb8-c48f-469b-a9bf-7bf6aa1ad101\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:06.229871954Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"failed to notify nexus about a new pool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:18:06.29544193Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"176.825657539s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"acf84034-6b1e-441c-9637-f3f88b291851\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Fri, 01 Mar 2024 23:18:06 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"acf84034-6b1e-441c-9637-f3f88b291851\\\" }\""} {"msg":"failed to notify nexus about a new pool oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:18:06.314273172Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"176.844892881s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"4b507e7a-a0a1-4891-b152-dcf11a276105\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Fri, 01 Mar 2024 23:18:06 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"4b507e7a-a0a1-4891-b152-dcf11a276105\\\" }\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: cc74b0a2-490e-4424-adad-bb6a277ade47, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:18:06.31585569Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"176.848267583s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"f5bc0e3e-9666-4be4-8c45-538028d43dd9\\\", \\\"content-length\\\": \\\"173\\\", \\\"date\\\": \\\"Fri, 01 Mar 2024 23:18:06 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: sled with id \\\\\\\"cc74b0a2-490e-4424-adad-bb6a277ade47\\\\\\\"\\\", request_id: \\\"f5bc0e3e-9666-4be4-8c45-538028d43dd9\\\" }\""} {"msg":"failed to notify nexus about a new pool oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:18:06.317595593Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"176.848680963s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"1e93b082-8398-4b58-bbaf-78a3a1daa947\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Fri, 01 Mar 2024 23:18:06 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"1e93b082-8398-4b58-bbaf-78a3a1daa947\\\" }\""} {"msg":"failed to notify nexus about a new pool oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:18:06.318820313Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"176.84977101s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"938902bd-d08d-4fab-9a1c-fc82123e6a2a\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Fri, 01 Mar 2024 23:18:06 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"938902bd-d08d-4fab-9a1c-fc82123e6a2a\\\" }\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", sled_id: cc74b0a2-490e-4424-adad-bb6a277ade47, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:18:06.322527664Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"176.855232934s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"8d9063f8-138d-471a-9e80-26e953102ea6\\\", \\\"content-length\\\": \\\"173\\\", \\\"date\\\": \\\"Fri, 01 Mar 2024 23:18:06 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: sled with id \\\\\\\"cc74b0a2-490e-4424-adad-bb6a277ade47\\\\\\\"\\\", request_id: \\\"8d9063f8-138d-471a-9e80-26e953102ea6\\\" }\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", sled_id: cc74b0a2-490e-4424-adad-bb6a277ade47, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:18:06.324986794Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"176.857874448s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"ef206bea-a53c-4a3c-8208-6feab65e936d\\\", \\\"content-length\\\": \\\"173\\\", \\\"date\\\": \\\"Fri, 01 Mar 2024 23:18:06 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: sled with id \\\\\\\"cc74b0a2-490e-4424-adad-bb6a277ade47\\\\\\\"\\\", request_id: \\\"ef206bea-a53c-4a3c-8208-6feab65e936d\\\" }\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: cc74b0a2-490e-4424-adad-bb6a277ade47, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:18:06.329079914Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"176.864890181s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"7fdfd035-fe8c-46cd-8566-151028b088ba\\\", \\\"content-length\\\": \\\"173\\\", \\\"date\\\": \\\"Fri, 01 Mar 2024 23:18:06 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: sled with id \\\\\\\"cc74b0a2-490e-4424-adad-bb6a277ade47\\\\\\\"\\\", request_id: \\\"7fdfd035-fe8c-46cd-8566-151028b088ba\\\" }\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: cc74b0a2-490e-4424-adad-bb6a277ade47, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:18:06.331340299Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"176.863511036s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"ae1958e9-bb56-4dae-98e3-acb2e8defa26\\\", \\\"content-length\\\": \\\"173\\\", \\\"date\\\": \\\"Fri, 01 Mar 2024 23:18:06 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: sled with id \\\\\\\"cc74b0a2-490e-4424-adad-bb6a277ade47\\\\\\\"\\\", request_id: \\\"ae1958e9-bb56-4dae-98e3-acb2e8defa26\\\" }\""} {"msg":"failed to notify nexus about a new pool oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:18:06.332678322Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"176.864015899s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"7f10c21d-1d0b-44a9-94ae-53df8e0ff891\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Fri, 01 Mar 2024 23:18:06 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"7f10c21d-1d0b-44a9-94ae-53df8e0ff891\\\" }\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: cc74b0a2-490e-4424-adad-bb6a277ade47, variant: M2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:18:06.335817299Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"176.868833167s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"3d6e6fa9-f252-40fa-a2fe-a052b6ed88ab\\\", \\\"content-length\\\": \\\"173\\\", \\\"date\\\": \\\"Fri, 01 Mar 2024 23:18:06 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: sled with id \\\\\\\"cc74b0a2-490e-4424-adad-bb6a277ade47\\\\\\\"\\\", request_id: \\\"3d6e6fa9-f252-40fa-a2fe-a052b6ed88ab\\\" }\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", sled_id: cc74b0a2-490e-4424-adad-bb6a277ade47, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:18:06.341327454Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"176.876482764s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"0eb25786-0b77-49a9-9118-6a6fccef6dd4\\\", \\\"content-length\\\": \\\"173\\\", \\\"date\\\": \\\"Fri, 01 Mar 2024 23:18:06 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: sled with id \\\\\\\"cc74b0a2-490e-4424-adad-bb6a277ade47\\\\\\\"\\\", request_id: \\\"0eb25786-0b77-49a9-9118-6a6fccef6dd4\\\" }\""} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:06.402020491Z","hostname":"buskin","pid":1671,"component":"HardwareMonitor","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"cc74b0a2-490e-4424-adad-bb6a277ade47"} {"msg":"failed to notify nexus about a new pool oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:18:06.425427224Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"176.956952225s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"e61015a9-40cf-4d11-980d-cb3365f04a26\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Fri, 01 Mar 2024 23:18:06 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"e61015a9-40cf-4d11-980d-cb3365f04a26\\\" }\""} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:06.488280433Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:33992"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:06.491327626Z","hostname":"buskin","pid":1671,"uri":"/network-bootstore-config","method":"GET","req_id":"b2bd8ccf-d863-43b5-b2f4-98e813e31a2c","remote_addr":"[fd00:1122:3344:101::b]:33992","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":3076,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:06.499826916Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:60589"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:06.49998007Z","hostname":"buskin","pid":1671,"uri":"/network-bootstore-config","method":"GET","req_id":"8f8dc181-ff9c-48cc-b194-f02dcd51d41a","remote_addr":"[fd00:1122:3344:101::c]:60589","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":114,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:06.510542988Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:47545"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:06.510683872Z","hostname":"buskin","pid":1671,"uri":"/network-bootstore-config","method":"GET","req_id":"23eaa4e8-f85b-4da2-bfb9-f237c733e1b8","remote_addr":"[fd00:1122:3344:101::a]:47545","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":51,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:06.539643761Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:56238"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:06.53999973Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"052fdaea-5fa4-4639-a45d-f4ef7289c0b0","remote_addr":"[fd00:1122:3344:101::b]:56238","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":337,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:06.541033665Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:06.541253371Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:06.543028344Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:06.543229019Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"16e873e9-6dce-48fd-b318-6350c94c2134","remote_addr":"[fd00:1122:3344:101::b]:56238","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":3060,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:06.548734194Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:63733"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:06.548831916Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"b347674d-8df6-49ef-ab6b-2c7a38c72c38","remote_addr":"[fd00:1122:3344:101::c]:63733","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":33,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:06.549018241Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:06.549036851Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:06.5510232Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:06.551379809Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"2b528d9e-3893-4bf7-9f2a-95c46658ea4b","remote_addr":"[fd00:1122:3344:101::c]:63733","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":2385,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:06.552856655Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:51752"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:06.553021789Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"3407843d-1df2-4693-a35b-6694ee605985","remote_addr":"[fd00:1122:3344:101::a]:51752","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":21,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:06.553221144Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:06.553230344Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:06.553390888Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:06.553594803Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"1f6f9eb3-610c-4a1f-9293-90381d958311","remote_addr":"[fd00:1122:3344:101::a]:51752","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":384,"response_code":"200"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3ffaedf8-ddef-42ff-b520-74a0bca6b772\", \"content-length\": \"174\", \"date\": \"Fri, 01 Mar 2024 23:18:06 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: zpool with id \\\"14b4dc87-ab46-49fb-a4b4-d361ae214c03\\\"\", request_id: \"3ffaedf8-ddef-42ff-b520-74a0bca6b772\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:06.942644143Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"890d0ccc-9f1a-44af-bf5d-34b053e30b2d\", \"content-length\": \"174\", \"date\": \"Fri, 01 Mar 2024 23:18:07 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: zpool with id \\\"14b4dc87-ab46-49fb-a4b4-d361ae214c03\\\"\", request_id: \"890d0ccc-9f1a-44af-bf5d-34b053e30b2d\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:07.510405661Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"failed to notify nexus about a new pool oxp_31bd71cd-4736-4a12-a387-9b74b050396f on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:18:07.920906126Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"178.452438199s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"7e0ea5af-0824-4c3c-b6e8-4a86bf49a0f6\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Fri, 01 Mar 2024 23:18:07 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"7e0ea5af-0824-4c3c-b6e8-4a86bf49a0f6\\\" }\""} {"msg":"failed to notify nexus about a new pool oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:18:07.922987957Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"178.454721836s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"d49b885d-c014-45a6-90d8-046c93dd9779\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Fri, 01 Mar 2024 23:18:07 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"d49b885d-c014-45a6-90d8-046c93dd9779\\\" }\""} {"msg":"failed to notify nexus about a new pool oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:18:07.967244001Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"178.498391895s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"efc8dc98-26be-4038-9f24-2ff02665c450\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Fri, 01 Mar 2024 23:18:07 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"efc8dc98-26be-4038-9f24-2ff02665c450\\\" }\""} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"825c4a8b-3237-469f-8ee3-578be9c83de9\", \"content-length\": \"174\", \"date\": \"Fri, 01 Mar 2024 23:18:07 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: zpool with id \\\"14b4dc87-ab46-49fb-a4b4-d361ae214c03\\\"\", request_id: \"825c4a8b-3237-469f-8ee3-578be9c83de9\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:08.280960196Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5badda5b-1474-49ff-84d7-16bf8a13b898\", \"content-length\": \"174\", \"date\": \"Fri, 01 Mar 2024 23:18:08 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: zpool with id \\\"14b4dc87-ab46-49fb-a4b4-d361ae214c03\\\"\", request_id: \"5badda5b-1474-49ff-84d7-16bf8a13b898\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:08.960570843Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: cc74b0a2-490e-4424-adad-bb6a277ade47, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:09.361106604Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"38dbfb88-0214-4917-a76d-a3f79bcab412\", \"content-length\": \"174\", \"date\": \"Fri, 01 Mar 2024 23:18:09 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: zpool with id \\\"14b4dc87-ab46-49fb-a4b4-d361ae214c03\\\"\", request_id: \"38dbfb88-0214-4917-a76d-a3f79bcab412\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:09.873934255Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:18:09.911439034Z","hostname":"buskin","pid":1671,"sled_id":"cc74b0a2-490e-4424-adad-bb6a277ade47","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1199","retry_after":"28.079993806s","error":"\"Metric registration error: Error Response: status: 503 Service Unavailable; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"57da3a16-c0fb-491b-9987-4fe7d91db343\\\", \\\"content-length\\\": \\\"133\\\", \\\"date\\\": \\\"Fri, 01 Mar 2024 23:18:08 GMT\\\"}; value: Error { error_code: Some(\\\"ServiceNotAvailable\\\"), message: \\\"Service Unavailable\\\", request_id: \\\"57da3a16-c0fb-491b-9987-4fe7d91db343\\\" }\""} {"msg":"failed to notify nexus about a new pool oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:18:10.073084894Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"180.60424826s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"36baa3e9-a241-45f3-87fd-7c2bf589ed64\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Fri, 01 Mar 2024 23:18:09 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"36baa3e9-a241-45f3-87fd-7c2bf589ed64\\\" }\""} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"297a0cdf-307e-4667-9915-e13a955a4c07\", \"content-length\": \"174\", \"date\": \"Fri, 01 Mar 2024 23:18:10 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: zpool with id \\\"14b4dc87-ab46-49fb-a4b4-d361ae214c03\\\"\", request_id: \"297a0cdf-307e-4667-9915-e13a955a4c07\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:10.833556171Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"17439918-92ef-47d6-9147-18e8234f9e5a\", \"content-length\": \"174\", \"date\": \"Fri, 01 Mar 2024 23:18:11 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: zpool with id \\\"14b4dc87-ab46-49fb-a4b4-d361ae214c03\\\"\", request_id: \"17439918-92ef-47d6-9147-18e8234f9e5a\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:11.524442824Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"188e6b25-415c-468e-9ce8-94c5ef1ca9c5\", \"content-length\": \"174\", \"date\": \"Fri, 01 Mar 2024 23:18:12 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: zpool with id \\\"14b4dc87-ab46-49fb-a4b4-d361ae214c03\\\"\", request_id: \"188e6b25-415c-468e-9ce8-94c5ef1ca9c5\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:12.606594252Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: cc74b0a2-490e-4424-adad-bb6a277ade47, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:12.627169126Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", sled_id: cc74b0a2-490e-4424-adad-bb6a277ade47, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:12.627185267Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", sled_id: cc74b0a2-490e-4424-adad-bb6a277ade47, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:12.627190687Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0bdbae98-4f50-4858-ae18-811e2a4511d8\", \"content-length\": \"174\", \"date\": \"Fri, 01 Mar 2024 23:18:13 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: zpool with id \\\"14b4dc87-ab46-49fb-a4b4-d361ae214c03\\\"\", request_id: \"0bdbae98-4f50-4858-ae18-811e2a4511d8\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:13.750075132Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: cc74b0a2-490e-4424-adad-bb6a277ade47, variant: M2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:14.041258305Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: cc74b0a2-490e-4424-adad-bb6a277ade47, variant: M2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:14.041282906Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", sled_id: cc74b0a2-490e-4424-adad-bb6a277ade47, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:14.041288275Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", sled_id: cc74b0a2-490e-4424-adad-bb6a277ade47, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:14.041292666Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: cc74b0a2-490e-4424-adad-bb6a277ade47, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:14.041297486Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: cc74b0a2-490e-4424-adad-bb6a277ade47, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:14.041301606Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: cc74b0a2-490e-4424-adad-bb6a277ade47, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:14.041305406Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:14.041314156Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"70d8ecd7-a122-4097-a4ef-4aa8b0ada6c3\", \"content-length\": \"174\", \"date\": \"Fri, 01 Mar 2024 23:18:14 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: zpool with id \\\"24b4dc87-ab46-49fb-a4b4-d361ae214c03\\\"\", request_id: \"70d8ecd7-a122-4097-a4ef-4aa8b0ada6c3\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:15.096838922Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:17.025931255Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:17.025960226Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:17.025966946Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:17.025972326Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:17.025977366Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:17.025983616Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:17.025989576Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:17.025994397Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:17.026000626Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:17.026022047Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:17.898211461Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:51586"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:17.900574869Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3af2d634-ca96-4144-9913-082ef495b783, kind: Service { id: 0a603215-e206-4aa0-8930-250f278a0d89 }, name: Name(\"external-dns-0a603215-e206-4aa0-8930-250f278a0d89\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 224, 81])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: d086f287-39ab-42aa-b1b5-4d91b0388531, kind: Service { id: 3f949516-3930-421a-bae2-b58ff4ea0dc8 }, name: Name(\"external-dns-3f949516-3930-421a-bae2-b58ff4ea0dc8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 146])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 06ae0dac-13b1-4ef0-995e-b56e4a7d3138, kind: Service { id: 933c0d3a-d49d-431b-8f65-51222f67e8f3 }, name: Name(\"nexus-933c0d3a-d49d-431b-8f65-51222f67e8f3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 46])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 9eeb19bd-897d-4d56-ae7e-bd0911ac9557, kind: Service { id: 264d5dd4-3eb2-4adb-b639-f0a26ab5534d }, name: Name(\"nexus-264d5dd4-3eb2-4adb-b639-f0a26ab5534d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 168, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: c6ce2abe-65af-4c28-bdee-3dca5bb68c72, kind: Service { id: 4ea96463-fca7-4b4a-8a70-4e88cc368897 }, name: Name(\"nexus-4ea96463-fca7-4b4a-8a70-4e88cc368897\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:17.9006203Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:17.900639531Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:17.900652411Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:17.900657861Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:17.900663301Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:17.900679302Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:17.900692002Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"f5a03d4c-d2a7-4aa5-934f-f0c1ecef0aca","remote_addr":"[fd00:1122:3344:101::c]:51586","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":2110,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1140eb2d-558f-425b-8f25-063f3f94fcd9\", \"content-length\": \"127\", \"date\": \"Fri, 01 Mar 2024 23:18:18 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"1140eb2d-558f-425b-8f25-063f3f94fcd9\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:18.019321378Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:18.197495222Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:55214"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:18.197575464Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"cbbd0c51-defe-44b1-b586-6597575beeeb","remote_addr":"[fd00:1122:3344:101::c]:55214","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":53,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:18.197777439Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:18.19780633Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:18.198150638Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:18.198494487Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"6681b6ed-e8a7-4ec4-bf42-6c4684e6cb4a","remote_addr":"[fd00:1122:3344:101::c]:55214","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":747,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:19.232904255Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:47938"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:19.233078819Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3af2d634-ca96-4144-9913-082ef495b783, kind: Service { id: 0a603215-e206-4aa0-8930-250f278a0d89 }, name: Name(\"external-dns-0a603215-e206-4aa0-8930-250f278a0d89\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 224, 81])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: d086f287-39ab-42aa-b1b5-4d91b0388531, kind: Service { id: 3f949516-3930-421a-bae2-b58ff4ea0dc8 }, name: Name(\"external-dns-3f949516-3930-421a-bae2-b58ff4ea0dc8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 146])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 06ae0dac-13b1-4ef0-995e-b56e4a7d3138, kind: Service { id: 933c0d3a-d49d-431b-8f65-51222f67e8f3 }, name: Name(\"nexus-933c0d3a-d49d-431b-8f65-51222f67e8f3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 46])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 9eeb19bd-897d-4d56-ae7e-bd0911ac9557, kind: Service { id: 264d5dd4-3eb2-4adb-b639-f0a26ab5534d }, name: Name(\"nexus-264d5dd4-3eb2-4adb-b639-f0a26ab5534d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 168, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: c6ce2abe-65af-4c28-bdee-3dca5bb68c72, kind: Service { id: 4ea96463-fca7-4b4a-8a70-4e88cc368897 }, name: Name(\"nexus-4ea96463-fca7-4b4a-8a70-4e88cc368897\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:19.23312288Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:19.233144811Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:19.233152071Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:19.233158681Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:19.233164371Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:19.233170401Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:19.233197092Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"f67fa3ac-e84d-4897-b8dc-698df7e500c9","remote_addr":"[fd00:1122:3344:101::c]:47938","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":243,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e860c195-1fd8-4ad1-926a-eacb9dfc0d01\", \"content-length\": \"127\", \"date\": \"Fri, 01 Mar 2024 23:18:19 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"e860c195-1fd8-4ad1-926a-eacb9dfc0d01\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:19.281334958Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:19.371065349Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:35397"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:19.371145081Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"ac887f5e-5aab-4abf-b0ed-56e9a518e937","remote_addr":"[fd00:1122:3344:101::c]:35397","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":41,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:19.371333636Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:19.371359876Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:19.371548811Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:19.371745536Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"1e83f674-56bb-4e51-846d-6ae60dbf0749","remote_addr":"[fd00:1122:3344:101::c]:35397","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":432,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:20.792272391Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:40010"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:20.792504397Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3af2d634-ca96-4144-9913-082ef495b783, kind: Service { id: 0a603215-e206-4aa0-8930-250f278a0d89 }, name: Name(\"external-dns-0a603215-e206-4aa0-8930-250f278a0d89\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 224, 81])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: d086f287-39ab-42aa-b1b5-4d91b0388531, kind: Service { id: 3f949516-3930-421a-bae2-b58ff4ea0dc8 }, name: Name(\"external-dns-3f949516-3930-421a-bae2-b58ff4ea0dc8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 146])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 06ae0dac-13b1-4ef0-995e-b56e4a7d3138, kind: Service { id: 933c0d3a-d49d-431b-8f65-51222f67e8f3 }, name: Name(\"nexus-933c0d3a-d49d-431b-8f65-51222f67e8f3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 46])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 9eeb19bd-897d-4d56-ae7e-bd0911ac9557, kind: Service { id: 264d5dd4-3eb2-4adb-b639-f0a26ab5534d }, name: Name(\"nexus-264d5dd4-3eb2-4adb-b639-f0a26ab5534d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 168, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: c6ce2abe-65af-4c28-bdee-3dca5bb68c72, kind: Service { id: 4ea96463-fca7-4b4a-8a70-4e88cc368897 }, name: Name(\"nexus-4ea96463-fca7-4b4a-8a70-4e88cc368897\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:20.792545908Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:20.792573208Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:20.792591839Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:20.792600339Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:20.792607729Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:20.792615609Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:20.79263695Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"d377ddd2-2d7d-49eb-8323-56ec019ef58d","remote_addr":"[fd00:1122:3344:101::c]:40010","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":285,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"18dc6ff0-76bf-440a-b56a-a5ffccc6b888\", \"content-length\": \"127\", \"date\": \"Fri, 01 Mar 2024 23:18:20 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"18dc6ff0-76bf-440a-b56a-a5ffccc6b888\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:20.850746563Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:20.943098846Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:51439"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:20.943174477Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"a5a3d43c-96ec-412d-9cf9-5a248441e970","remote_addr":"[fd00:1122:3344:101::c]:51439","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":38,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:20.943326361Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:20.943354392Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:20.943539536Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:20.94370265Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"0824c2c7-b27d-47eb-9da5-56db94d88b8c","remote_addr":"[fd00:1122:3344:101::c]:51439","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":393,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:23.055629673Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:49877"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:23.055776766Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3af2d634-ca96-4144-9913-082ef495b783, kind: Service { id: 0a603215-e206-4aa0-8930-250f278a0d89 }, name: Name(\"external-dns-0a603215-e206-4aa0-8930-250f278a0d89\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 224, 81])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: d086f287-39ab-42aa-b1b5-4d91b0388531, kind: Service { id: 3f949516-3930-421a-bae2-b58ff4ea0dc8 }, name: Name(\"external-dns-3f949516-3930-421a-bae2-b58ff4ea0dc8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 146])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 06ae0dac-13b1-4ef0-995e-b56e4a7d3138, kind: Service { id: 933c0d3a-d49d-431b-8f65-51222f67e8f3 }, name: Name(\"nexus-933c0d3a-d49d-431b-8f65-51222f67e8f3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 46])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 9eeb19bd-897d-4d56-ae7e-bd0911ac9557, kind: Service { id: 264d5dd4-3eb2-4adb-b639-f0a26ab5534d }, name: Name(\"nexus-264d5dd4-3eb2-4adb-b639-f0a26ab5534d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 168, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: c6ce2abe-65af-4c28-bdee-3dca5bb68c72, kind: Service { id: 4ea96463-fca7-4b4a-8a70-4e88cc368897 }, name: Name(\"nexus-4ea96463-fca7-4b4a-8a70-4e88cc368897\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:23.055811027Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:23.055821527Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:23.055829708Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:23.055836098Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:23.055841448Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:23.055852408Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:23.055876389Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"35c0e7eb-539c-4e8c-a8d8-282640088e8d","remote_addr":"[fd00:1122:3344:101::c]:49877","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":242,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"179e1392-f468-4236-99fb-f780b7bf3cd2\", \"content-length\": \"127\", \"date\": \"Fri, 01 Mar 2024 23:18:23 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"179e1392-f468-4236-99fb-f780b7bf3cd2\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:23.10816553Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:23.214474564Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:46630"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:23.214540546Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"9a4fbbd7-ac08-4292-b3b8-223c9234e0a9","remote_addr":"[fd00:1122:3344:101::c]:46630","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":42,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:23.215500619Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:23.21551677Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:23.215706854Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:23.215869868Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"9eff1cbf-050a-4cb7-8e55-7ddd0ca7817b","remote_addr":"[fd00:1122:3344:101::c]:46630","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":1191,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:26.602865835Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:51183"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:26.603075372Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3af2d634-ca96-4144-9913-082ef495b783, kind: Service { id: 0a603215-e206-4aa0-8930-250f278a0d89 }, name: Name(\"external-dns-0a603215-e206-4aa0-8930-250f278a0d89\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 224, 81])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: d086f287-39ab-42aa-b1b5-4d91b0388531, kind: Service { id: 3f949516-3930-421a-bae2-b58ff4ea0dc8 }, name: Name(\"external-dns-3f949516-3930-421a-bae2-b58ff4ea0dc8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 146])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 06ae0dac-13b1-4ef0-995e-b56e4a7d3138, kind: Service { id: 933c0d3a-d49d-431b-8f65-51222f67e8f3 }, name: Name(\"nexus-933c0d3a-d49d-431b-8f65-51222f67e8f3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 46])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 9eeb19bd-897d-4d56-ae7e-bd0911ac9557, kind: Service { id: 264d5dd4-3eb2-4adb-b639-f0a26ab5534d }, name: Name(\"nexus-264d5dd4-3eb2-4adb-b639-f0a26ab5534d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 168, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: c6ce2abe-65af-4c28-bdee-3dca5bb68c72, kind: Service { id: 4ea96463-fca7-4b4a-8a70-4e88cc368897 }, name: Name(\"nexus-4ea96463-fca7-4b4a-8a70-4e88cc368897\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:26.603119895Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:26.603173146Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:26.603183965Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:26.603192994Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:26.603201309Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:26.603210094Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:26.603233467Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"a5df8c89-d631-4809-ab31-ade074208bb7","remote_addr":"[fd00:1122:3344:101::c]:51183","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":364,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"943a3337-e3ec-4310-af5b-cc65b34c80e6\", \"content-length\": \"127\", \"date\": \"Fri, 01 Mar 2024 23:18:26 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"943a3337-e3ec-4310-af5b-cc65b34c80e6\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:26.652241462Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:26.736162368Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:65214"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:26.736301671Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"4e299f06-30d8-4949-a32d-3ff68e25bdb6","remote_addr":"[fd00:1122:3344:101::c]:65214","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":32,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:26.737024079Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:26.737048959Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:26.737243354Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:26.737411518Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"5c94c67e-6414-4e94-9a9c-123f08a3a298","remote_addr":"[fd00:1122:3344:101::c]:65214","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":890,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:30.006056484Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:62690"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:30.006257769Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3af2d634-ca96-4144-9913-082ef495b783, kind: Service { id: 0a603215-e206-4aa0-8930-250f278a0d89 }, name: Name(\"external-dns-0a603215-e206-4aa0-8930-250f278a0d89\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 224, 81])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: d086f287-39ab-42aa-b1b5-4d91b0388531, kind: Service { id: 3f949516-3930-421a-bae2-b58ff4ea0dc8 }, name: Name(\"external-dns-3f949516-3930-421a-bae2-b58ff4ea0dc8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 146])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 06ae0dac-13b1-4ef0-995e-b56e4a7d3138, kind: Service { id: 933c0d3a-d49d-431b-8f65-51222f67e8f3 }, name: Name(\"nexus-933c0d3a-d49d-431b-8f65-51222f67e8f3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 46])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 9eeb19bd-897d-4d56-ae7e-bd0911ac9557, kind: Service { id: 264d5dd4-3eb2-4adb-b639-f0a26ab5534d }, name: Name(\"nexus-264d5dd4-3eb2-4adb-b639-f0a26ab5534d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 168, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: c6ce2abe-65af-4c28-bdee-3dca5bb68c72, kind: Service { id: 4ea96463-fca7-4b4a-8a70-4e88cc368897 }, name: Name(\"nexus-4ea96463-fca7-4b4a-8a70-4e88cc368897\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:30.0063026Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:30.0063297Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:30.006336691Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:30.006343861Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:30.006348971Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:30.006356001Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:30.006370111Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"aec4b489-bb29-4313-b6ad-2382ee0d48d4","remote_addr":"[fd00:1122:3344:101::c]:62690","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":240,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c1ee4c49-3be9-4ffa-97d0-4abdd4e4426a\", \"content-length\": \"127\", \"date\": \"Fri, 01 Mar 2024 23:18:30 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"c1ee4c49-3be9-4ffa-97d0-4abdd4e4426a\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:30.059017581Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:30.148078152Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:62547"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:30.148191505Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"b8a601ee-d2bc-494a-8b4d-cd14b3c593ea","remote_addr":"[fd00:1122:3344:101::c]:62547","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":34,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:30.148830061Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:30.148848542Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:30.149062287Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:30.149231331Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"24a1439d-b6d5-4d18-8b41-cf8a26c9dad0","remote_addr":"[fd00:1122:3344:101::c]:62547","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":829,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:34.589521137Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:63657"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:34.590083301Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"5623c87a-1d9f-4686-86d4-a4f5c84d58c5","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:63657","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":37,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:36.986586547Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:49022"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:36.986789983Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3af2d634-ca96-4144-9913-082ef495b783, kind: Service { id: 0a603215-e206-4aa0-8930-250f278a0d89 }, name: Name(\"external-dns-0a603215-e206-4aa0-8930-250f278a0d89\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 224, 81])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: d086f287-39ab-42aa-b1b5-4d91b0388531, kind: Service { id: 3f949516-3930-421a-bae2-b58ff4ea0dc8 }, name: Name(\"external-dns-3f949516-3930-421a-bae2-b58ff4ea0dc8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 146])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 06ae0dac-13b1-4ef0-995e-b56e4a7d3138, kind: Service { id: 933c0d3a-d49d-431b-8f65-51222f67e8f3 }, name: Name(\"nexus-933c0d3a-d49d-431b-8f65-51222f67e8f3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 46])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 9eeb19bd-897d-4d56-ae7e-bd0911ac9557, kind: Service { id: 264d5dd4-3eb2-4adb-b639-f0a26ab5534d }, name: Name(\"nexus-264d5dd4-3eb2-4adb-b639-f0a26ab5534d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 168, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: c6ce2abe-65af-4c28-bdee-3dca5bb68c72, kind: Service { id: 4ea96463-fca7-4b4a-8a70-4e88cc368897 }, name: Name(\"nexus-4ea96463-fca7-4b4a-8a70-4e88cc368897\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:36.986828463Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:36.986850234Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:36.986858694Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:36.986865374Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:36.986885975Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:36.986893845Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:36.986910485Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"0586a10e-7a60-42b5-8c26-3bac60b6f322","remote_addr":"[fd00:1122:3344:101::c]:49022","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":251,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"bd2a085d-59be-41ee-bd2e-a568766474b4\", \"content-length\": \"127\", \"date\": \"Fri, 01 Mar 2024 23:18:37 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"bd2a085d-59be-41ee-bd2e-a568766474b4\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:37.033532768Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:37.121302028Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:50547"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:37.12139149Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"53e5e393-ecb2-4a4b-bc60-13e5fcd9b4d6","remote_addr":"[fd00:1122:3344:101::c]:50547","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":46,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:37.122075466Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:37.122096397Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:37.122305572Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:37.122483827Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"e2c5f60e-4857-43c8-8043-7f06ab5bba2c","remote_addr":"[fd00:1122:3344:101::c]:50547","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":966,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:40.724015672Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:60804"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:40.724193356Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3af2d634-ca96-4144-9913-082ef495b783, kind: Service { id: 0a603215-e206-4aa0-8930-250f278a0d89 }, name: Name(\"external-dns-0a603215-e206-4aa0-8930-250f278a0d89\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 224, 81])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: d086f287-39ab-42aa-b1b5-4d91b0388531, kind: Service { id: 3f949516-3930-421a-bae2-b58ff4ea0dc8 }, name: Name(\"external-dns-3f949516-3930-421a-bae2-b58ff4ea0dc8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 146])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 06ae0dac-13b1-4ef0-995e-b56e4a7d3138, kind: Service { id: 933c0d3a-d49d-431b-8f65-51222f67e8f3 }, name: Name(\"nexus-933c0d3a-d49d-431b-8f65-51222f67e8f3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 46])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 9eeb19bd-897d-4d56-ae7e-bd0911ac9557, kind: Service { id: 264d5dd4-3eb2-4adb-b639-f0a26ab5534d }, name: Name(\"nexus-264d5dd4-3eb2-4adb-b639-f0a26ab5534d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 168, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: c6ce2abe-65af-4c28-bdee-3dca5bb68c72, kind: Service { id: 4ea96463-fca7-4b4a-8a70-4e88cc368897 }, name: Name(\"nexus-4ea96463-fca7-4b4a-8a70-4e88cc368897\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:40.724236557Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:40.724264258Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:40.724271108Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:40.724290648Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:40.724296308Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:40.724313129Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:40.724318839Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"2a3f7acf-181a-4b4e-ab4c-ae1c2e39d2b2","remote_addr":"[fd00:1122:3344:101::c]:60804","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":242,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b2f73e7d-b51c-418f-9627-f03b9237f1f1\", \"content-length\": \"127\", \"date\": \"Fri, 01 Mar 2024 23:18:40 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"b2f73e7d-b51c-418f-9627-f03b9237f1f1\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:40.786033341Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:40.89378487Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:62344"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:40.893852332Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"8aea8cd4-f1b1-4eb6-83a5-0e1b80202174","remote_addr":"[fd00:1122:3344:101::c]:62344","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":42,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:40.893981255Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:40.894004206Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:40.894213551Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:40.894405495Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"1bbb26ba-003a-43c7-930a-95cdfbec9b7c","remote_addr":"[fd00:1122:3344:101::c]:62344","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":445,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:44.85151244Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:49017"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:44.851675214Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3af2d634-ca96-4144-9913-082ef495b783, kind: Service { id: 0a603215-e206-4aa0-8930-250f278a0d89 }, name: Name(\"external-dns-0a603215-e206-4aa0-8930-250f278a0d89\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 224, 81])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: d086f287-39ab-42aa-b1b5-4d91b0388531, kind: Service { id: 3f949516-3930-421a-bae2-b58ff4ea0dc8 }, name: Name(\"external-dns-3f949516-3930-421a-bae2-b58ff4ea0dc8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 146])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 06ae0dac-13b1-4ef0-995e-b56e4a7d3138, kind: Service { id: 933c0d3a-d49d-431b-8f65-51222f67e8f3 }, name: Name(\"nexus-933c0d3a-d49d-431b-8f65-51222f67e8f3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 46])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 9eeb19bd-897d-4d56-ae7e-bd0911ac9557, kind: Service { id: 264d5dd4-3eb2-4adb-b639-f0a26ab5534d }, name: Name(\"nexus-264d5dd4-3eb2-4adb-b639-f0a26ab5534d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 168, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: c6ce2abe-65af-4c28-bdee-3dca5bb68c72, kind: Service { id: 4ea96463-fca7-4b4a-8a70-4e88cc368897 }, name: Name(\"nexus-4ea96463-fca7-4b4a-8a70-4e88cc368897\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:44.851718255Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:44.851745256Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:44.851778616Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:44.851784077Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:44.851789057Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:44.851794627Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:44.851800327Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"495053a1-47dd-47e0-95e0-35e45af21c6c","remote_addr":"[fd00:1122:3344:101::c]:49017","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":252,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ec8490ae-783a-41f5-bf81-e66566a88847\", \"content-length\": \"127\", \"date\": \"Fri, 01 Mar 2024 23:18:44 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"ec8490ae-783a-41f5-bf81-e66566a88847\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:44.902274033Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:45.002231832Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:54294"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:45.002318094Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"9e79ff46-9a87-4ce5-b2d0-13b4991fc74f","remote_addr":"[fd00:1122:3344:101::c]:54294","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":47,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:45.002486868Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:45.002521409Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:45.002745955Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:45.002930589Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"5579aaee-3e24-414a-bbe4-65cf3cdc2179","remote_addr":"[fd00:1122:3344:101::c]:54294","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":475,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:55.139642804Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:58930"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:55.139781848Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3af2d634-ca96-4144-9913-082ef495b783, kind: Service { id: 0a603215-e206-4aa0-8930-250f278a0d89 }, name: Name(\"external-dns-0a603215-e206-4aa0-8930-250f278a0d89\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 224, 81])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: d086f287-39ab-42aa-b1b5-4d91b0388531, kind: Service { id: 3f949516-3930-421a-bae2-b58ff4ea0dc8 }, name: Name(\"external-dns-3f949516-3930-421a-bae2-b58ff4ea0dc8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 146])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 06ae0dac-13b1-4ef0-995e-b56e4a7d3138, kind: Service { id: 933c0d3a-d49d-431b-8f65-51222f67e8f3 }, name: Name(\"nexus-933c0d3a-d49d-431b-8f65-51222f67e8f3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 46])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 9eeb19bd-897d-4d56-ae7e-bd0911ac9557, kind: Service { id: 264d5dd4-3eb2-4adb-b639-f0a26ab5534d }, name: Name(\"nexus-264d5dd4-3eb2-4adb-b639-f0a26ab5534d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 168, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: c6ce2abe-65af-4c28-bdee-3dca5bb68c72, kind: Service { id: 4ea96463-fca7-4b4a-8a70-4e88cc368897 }, name: Name(\"nexus-4ea96463-fca7-4b4a-8a70-4e88cc368897\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:55.139818378Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:55.139840919Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:55.139847589Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:55.139853089Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:55.139858339Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:55.139864119Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:55.13988031Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"6344155b-9557-46db-bafd-61ad7c6e984d","remote_addr":"[fd00:1122:3344:101::c]:58930","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":220,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4560e504-3f34-49cd-af4a-58384a3f684b\", \"content-length\": \"127\", \"date\": \"Fri, 01 Mar 2024 23:18:55 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"4560e504-3f34-49cd-af4a-58384a3f684b\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:55.195417131Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:55.296151758Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:40971"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:55.29621242Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"5070a2d8-a1f9-43db-b614-4d0c17d191b8","remote_addr":"[fd00:1122:3344:101::c]:40971","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":41,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:55.296389824Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:55.296410894Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:55.296666761Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:18:55.296845755Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"82fa1465-029b-4698-bbe3-bf96efd74fec","remote_addr":"[fd00:1122:3344:101::c]:40971","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":475,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:19:02.43155314Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:60475"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:19:02.432591916Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3af2d634-ca96-4144-9913-082ef495b783, kind: Service { id: 0a603215-e206-4aa0-8930-250f278a0d89 }, name: Name(\"external-dns-0a603215-e206-4aa0-8930-250f278a0d89\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 224, 81])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: d086f287-39ab-42aa-b1b5-4d91b0388531, kind: Service { id: 3f949516-3930-421a-bae2-b58ff4ea0dc8 }, name: Name(\"external-dns-3f949516-3930-421a-bae2-b58ff4ea0dc8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 146])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 06ae0dac-13b1-4ef0-995e-b56e4a7d3138, kind: Service { id: 933c0d3a-d49d-431b-8f65-51222f67e8f3 }, name: Name(\"nexus-933c0d3a-d49d-431b-8f65-51222f67e8f3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 46])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 9eeb19bd-897d-4d56-ae7e-bd0911ac9557, kind: Service { id: 264d5dd4-3eb2-4adb-b639-f0a26ab5534d }, name: Name(\"nexus-264d5dd4-3eb2-4adb-b639-f0a26ab5534d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 168, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: c6ce2abe-65af-4c28-bdee-3dca5bb68c72, kind: Service { id: 4ea96463-fca7-4b4a-8a70-4e88cc368897 }, name: Name(\"nexus-4ea96463-fca7-4b4a-8a70-4e88cc368897\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:19:02.432619906Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:19:02.432649557Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:19:02.432657027Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:19:02.432665428Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:19:02.432670708Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:19:02.432677648Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:19:02.432701008Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"695e6b1e-5cd7-41ef-9abd-26785c5260f9","remote_addr":"[fd00:1122:3344:101::c]:60475","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":1062,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"20201508-708d-4d28-9200-1af73aaeb601\", \"content-length\": \"127\", \"date\": \"Fri, 01 Mar 2024 23:19:02 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"20201508-708d-4d28-9200-1af73aaeb601\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:19:02.493405655Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:19:02.596780918Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:57421"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:19:02.59685108Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"aeddc6e5-e940-4d05-9a50-8d3c18949c1f","remote_addr":"[fd00:1122:3344:101::c]:57421","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":45,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:19:02.597023664Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:19:02.597044934Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:19:02.59728236Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:19:02.597502996Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"9cd111a9-5dac-49db-af12-c85258f166d1","remote_addr":"[fd00:1122:3344:101::c]:57421","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":502,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:19:04.622009318Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:33477"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:19:04.622048419Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"5d672441-acd8-4aa6-ab59-b0f38539c99f","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:33477","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":32,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:19:08.070160846Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::d]:60361"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:19:08.076330287Z","hostname":"buskin","pid":1671,"uri":"/metrics/collect/cc74b0a2-490e-4424-adad-bb6a277ade47","method":"GET","req_id":"0442e6ef-92a6-4705-bf51-591118003c11","remote_addr":"[fd00:1122:3344:101::d]:60361","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":6139,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:19:19.352280419Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:53929"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:19:19.352455194Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3af2d634-ca96-4144-9913-082ef495b783, kind: Service { id: 0a603215-e206-4aa0-8930-250f278a0d89 }, name: Name(\"external-dns-0a603215-e206-4aa0-8930-250f278a0d89\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 224, 81])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: d086f287-39ab-42aa-b1b5-4d91b0388531, kind: Service { id: 3f949516-3930-421a-bae2-b58ff4ea0dc8 }, name: Name(\"external-dns-3f949516-3930-421a-bae2-b58ff4ea0dc8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 146])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 06ae0dac-13b1-4ef0-995e-b56e4a7d3138, kind: Service { id: 933c0d3a-d49d-431b-8f65-51222f67e8f3 }, name: Name(\"nexus-933c0d3a-d49d-431b-8f65-51222f67e8f3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 46])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 9eeb19bd-897d-4d56-ae7e-bd0911ac9557, kind: Service { id: 264d5dd4-3eb2-4adb-b639-f0a26ab5534d }, name: Name(\"nexus-264d5dd4-3eb2-4adb-b639-f0a26ab5534d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 168, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: c6ce2abe-65af-4c28-bdee-3dca5bb68c72, kind: Service { id: 4ea96463-fca7-4b4a-8a70-4e88cc368897 }, name: Name(\"nexus-4ea96463-fca7-4b4a-8a70-4e88cc368897\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:19:19.352499324Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:19:19.352512415Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:19:19.352520915Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:19:19.352528925Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:19:19.352535765Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:19:19.352546306Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:19:19.352567276Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"f889c588-8d28-45d4-b620-7d3a026bf62a","remote_addr":"[fd00:1122:3344:101::c]:53929","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":240,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"169dafef-f7c8-4c81-b093-1801a51af224\", \"content-length\": \"127\", \"date\": \"Fri, 01 Mar 2024 23:19:19 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"169dafef-f7c8-4c81-b093-1801a51af224\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:19:19.407126443Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:19:19.494572934Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:37205"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:19:19.494655796Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"8e7b68b1-a635-45f9-bc49-6c73630337bf","remote_addr":"[fd00:1122:3344:101::c]:37205","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":36,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:19:19.49482719Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:19:19.494845991Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:19:19.495091237Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:19:19.495281261Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"161bba96-c193-40f6-8c77-b21c0a2243ba","remote_addr":"[fd00:1122:3344:101::c]:37205","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":472,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:19:29.005990971Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:45374"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:19:29.006825961Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3af2d634-ca96-4144-9913-082ef495b783, kind: Service { id: 0a603215-e206-4aa0-8930-250f278a0d89 }, name: Name(\"external-dns-0a603215-e206-4aa0-8930-250f278a0d89\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 224, 81])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: d086f287-39ab-42aa-b1b5-4d91b0388531, kind: Service { id: 3f949516-3930-421a-bae2-b58ff4ea0dc8 }, name: Name(\"external-dns-3f949516-3930-421a-bae2-b58ff4ea0dc8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 146])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 06ae0dac-13b1-4ef0-995e-b56e4a7d3138, kind: Service { id: 933c0d3a-d49d-431b-8f65-51222f67e8f3 }, name: Name(\"nexus-933c0d3a-d49d-431b-8f65-51222f67e8f3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 46])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 9eeb19bd-897d-4d56-ae7e-bd0911ac9557, kind: Service { id: 264d5dd4-3eb2-4adb-b639-f0a26ab5534d }, name: Name(\"nexus-264d5dd4-3eb2-4adb-b639-f0a26ab5534d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 168, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: c6ce2abe-65af-4c28-bdee-3dca5bb68c72, kind: Service { id: 4ea96463-fca7-4b4a-8a70-4e88cc368897 }, name: Name(\"nexus-4ea96463-fca7-4b4a-8a70-4e88cc368897\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:19:29.006862222Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:19:29.006882803Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:19:29.006889943Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:19:29.006895763Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:19:29.006901713Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:19:29.006911414Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:19:29.006925684Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"103b5952-ca16-46df-aa8e-bb04ef40fe7f","remote_addr":"[fd00:1122:3344:101::c]:45374","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":885,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"97844887-5748-4293-82ca-34f8e9a9ac28\", \"content-length\": \"127\", \"date\": \"Fri, 01 Mar 2024 23:19:29 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"97844887-5748-4293-82ca-34f8e9a9ac28\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:19:29.057632456Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:19:29.143431198Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:33435"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:19:29.144037203Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"5adeead2-7d61-4086-a9a4-5dbc58aef1af","remote_addr":"[fd00:1122:3344:101::c]:33435","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":576,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:19:29.144204497Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:19:29.144230278Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:19:29.144412022Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:19:29.144579506Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"98d19251-bca6-45dd-a9ca-f9756bb1c977","remote_addr":"[fd00:1122:3344:101::c]:33435","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":397,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:19:34.65450599Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:64001"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:19:34.654584332Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"6f4a93a3-8359-4b6f-97b6-511f8586e632","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:64001","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":47,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:19:38.069299541Z","hostname":"buskin","pid":1671,"uri":"/metrics/collect/cc74b0a2-490e-4424-adad-bb6a277ade47","method":"GET","req_id":"b0dba87a-288b-4d9b-88aa-f58a442f4932","remote_addr":"[fd00:1122:3344:101::d]:60361","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":207,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:19:43.236020047Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:65182"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:19:43.236813826Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3af2d634-ca96-4144-9913-082ef495b783, kind: Service { id: 0a603215-e206-4aa0-8930-250f278a0d89 }, name: Name(\"external-dns-0a603215-e206-4aa0-8930-250f278a0d89\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 224, 81])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: d086f287-39ab-42aa-b1b5-4d91b0388531, kind: Service { id: 3f949516-3930-421a-bae2-b58ff4ea0dc8 }, name: Name(\"external-dns-3f949516-3930-421a-bae2-b58ff4ea0dc8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 146])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 06ae0dac-13b1-4ef0-995e-b56e4a7d3138, kind: Service { id: 933c0d3a-d49d-431b-8f65-51222f67e8f3 }, name: Name(\"nexus-933c0d3a-d49d-431b-8f65-51222f67e8f3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 46])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 9eeb19bd-897d-4d56-ae7e-bd0911ac9557, kind: Service { id: 264d5dd4-3eb2-4adb-b639-f0a26ab5534d }, name: Name(\"nexus-264d5dd4-3eb2-4adb-b639-f0a26ab5534d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 168, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: c6ce2abe-65af-4c28-bdee-3dca5bb68c72, kind: Service { id: 4ea96463-fca7-4b4a-8a70-4e88cc368897 }, name: Name(\"nexus-4ea96463-fca7-4b4a-8a70-4e88cc368897\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:19:43.236865748Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:19:43.236880368Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:19:43.236889188Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:19:43.236894599Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:19:43.236900489Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:19:43.236906859Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:19:43.236929159Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"ee48ba2e-3225-449b-96a8-1402abe790b4","remote_addr":"[fd00:1122:3344:101::c]:65182","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":884,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d03121fa-cd5c-4cc2-8644-3136b173947a\", \"content-length\": \"127\", \"date\": \"Fri, 01 Mar 2024 23:19:43 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"d03121fa-cd5c-4cc2-8644-3136b173947a\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:19:43.302361582Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:19:43.407322223Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:55589"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:19:43.407368335Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"48758646-2aac-4325-b40b-e7fac2eb1130","remote_addr":"[fd00:1122:3344:101::c]:55589","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":36,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:19:43.407508608Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:19:43.407524359Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:19:43.407764544Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:19:43.407944149Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"13997bc7-00b6-4796-914d-f37ed93cff88","remote_addr":"[fd00:1122:3344:101::c]:55589","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":458,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:20:04.691136347Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:55976"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:20:04.691172368Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"548a5b2e-26b1-4bde-becd-b48af440e8d5","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:55976","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":27,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:20:08.069638719Z","hostname":"buskin","pid":1671,"uri":"/metrics/collect/cc74b0a2-490e-4424-adad-bb6a277ade47","method":"GET","req_id":"024472b1-6f00-43f9-b2f3-857f867ec6a7","remote_addr":"[fd00:1122:3344:101::d]:60361","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":302,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:20:12.43401187Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:63762"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:20:12.434271316Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3af2d634-ca96-4144-9913-082ef495b783, kind: Service { id: 0a603215-e206-4aa0-8930-250f278a0d89 }, name: Name(\"external-dns-0a603215-e206-4aa0-8930-250f278a0d89\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 224, 81])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: d086f287-39ab-42aa-b1b5-4d91b0388531, kind: Service { id: 3f949516-3930-421a-bae2-b58ff4ea0dc8 }, name: Name(\"external-dns-3f949516-3930-421a-bae2-b58ff4ea0dc8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 146])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 06ae0dac-13b1-4ef0-995e-b56e4a7d3138, kind: Service { id: 933c0d3a-d49d-431b-8f65-51222f67e8f3 }, name: Name(\"nexus-933c0d3a-d49d-431b-8f65-51222f67e8f3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 46])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 9eeb19bd-897d-4d56-ae7e-bd0911ac9557, kind: Service { id: 264d5dd4-3eb2-4adb-b639-f0a26ab5534d }, name: Name(\"nexus-264d5dd4-3eb2-4adb-b639-f0a26ab5534d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 168, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: c6ce2abe-65af-4c28-bdee-3dca5bb68c72, kind: Service { id: 4ea96463-fca7-4b4a-8a70-4e88cc368897 }, name: Name(\"nexus-4ea96463-fca7-4b4a-8a70-4e88cc368897\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:20:12.434329198Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:20:12.434363249Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:20:12.434370939Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:20:12.434379489Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:20:12.434388289Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:20:12.43440015Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:20:12.43443948Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"9808ef35-59ad-4784-99c2-394a87595359","remote_addr":"[fd00:1122:3344:101::c]:63762","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":341,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"00b2ed98-dcc6-4701-9622-16430efbb7ce\", \"content-length\": \"127\", \"date\": \"Fri, 01 Mar 2024 23:20:12 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"00b2ed98-dcc6-4701-9622-16430efbb7ce\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:20:12.501476422Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:20:12.604044715Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:64420"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:20:12.60508274Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"127a8f56-7690-4445-8a58-1a565118a0b3","remote_addr":"[fd00:1122:3344:101::c]:64420","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":1006,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:20:12.605207944Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:20:12.605232704Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:20:12.60547535Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:20:12.605648534Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"dc6a6a62-31d1-41c1-8afa-498a5acab33f","remote_addr":"[fd00:1122:3344:101::c]:64420","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":460,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:20:32.699249093Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:46657"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:20:32.699977911Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3af2d634-ca96-4144-9913-082ef495b783, kind: Service { id: 0a603215-e206-4aa0-8930-250f278a0d89 }, name: Name(\"external-dns-0a603215-e206-4aa0-8930-250f278a0d89\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 224, 81])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: d086f287-39ab-42aa-b1b5-4d91b0388531, kind: Service { id: 3f949516-3930-421a-bae2-b58ff4ea0dc8 }, name: Name(\"external-dns-3f949516-3930-421a-bae2-b58ff4ea0dc8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 146])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 06ae0dac-13b1-4ef0-995e-b56e4a7d3138, kind: Service { id: 933c0d3a-d49d-431b-8f65-51222f67e8f3 }, name: Name(\"nexus-933c0d3a-d49d-431b-8f65-51222f67e8f3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 46])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 9eeb19bd-897d-4d56-ae7e-bd0911ac9557, kind: Service { id: 264d5dd4-3eb2-4adb-b639-f0a26ab5534d }, name: Name(\"nexus-264d5dd4-3eb2-4adb-b639-f0a26ab5534d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 168, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: c6ce2abe-65af-4c28-bdee-3dca5bb68c72, kind: Service { id: 4ea96463-fca7-4b4a-8a70-4e88cc368897 }, name: Name(\"nexus-4ea96463-fca7-4b4a-8a70-4e88cc368897\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:20:32.700014892Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:20:32.700039053Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:20:32.700048943Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:20:32.700058403Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:20:32.700067604Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:20:32.700091394Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:20:32.700100484Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"d451d6b8-62bc-4cd4-ac98-f7c849c8638c","remote_addr":"[fd00:1122:3344:101::c]:46657","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":246,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"14000c00-c9db-4269-84e1-b68295a889c0\", \"content-length\": \"127\", \"date\": \"Fri, 01 Mar 2024 23:20:32 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"14000c00-c9db-4269-84e1-b68295a889c0\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:20:32.757290285Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:20:32.880718299Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:35346"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:20:32.880790771Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"a61eeb52-2b81-4d87-9831-d5f44062fcbe","remote_addr":"[fd00:1122:3344:101::c]:35346","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":47,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:20:32.880985296Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:20:32.882249396Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:20:32.882469322Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:20:32.882665597Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"ab5b6e6f-6836-4b62-8396-20737a57f07b","remote_addr":"[fd00:1122:3344:101::c]:35346","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":1713,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:20:34.723008818Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:35257"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:20:34.72309038Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"2bf73290-9656-4ebd-8d32-5b178cb2f917","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:35257","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":45,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:20:38.06970247Z","hostname":"buskin","pid":1671,"uri":"/metrics/collect/cc74b0a2-490e-4424-adad-bb6a277ade47","method":"GET","req_id":"61c5bfbf-ad3b-41bd-8d15-15051266090a","remote_addr":"[fd00:1122:3344:101::d]:60361","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":260,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:20:52.806931916Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:36377"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:20:52.80709771Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3af2d634-ca96-4144-9913-082ef495b783, kind: Service { id: 0a603215-e206-4aa0-8930-250f278a0d89 }, name: Name(\"external-dns-0a603215-e206-4aa0-8930-250f278a0d89\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 224, 81])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: d086f287-39ab-42aa-b1b5-4d91b0388531, kind: Service { id: 3f949516-3930-421a-bae2-b58ff4ea0dc8 }, name: Name(\"external-dns-3f949516-3930-421a-bae2-b58ff4ea0dc8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 146])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 06ae0dac-13b1-4ef0-995e-b56e4a7d3138, kind: Service { id: 933c0d3a-d49d-431b-8f65-51222f67e8f3 }, name: Name(\"nexus-933c0d3a-d49d-431b-8f65-51222f67e8f3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 46])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 9eeb19bd-897d-4d56-ae7e-bd0911ac9557, kind: Service { id: 264d5dd4-3eb2-4adb-b639-f0a26ab5534d }, name: Name(\"nexus-264d5dd4-3eb2-4adb-b639-f0a26ab5534d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 168, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: c6ce2abe-65af-4c28-bdee-3dca5bb68c72, kind: Service { id: 4ea96463-fca7-4b4a-8a70-4e88cc368897 }, name: Name(\"nexus-4ea96463-fca7-4b4a-8a70-4e88cc368897\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:20:52.807137291Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:20:52.807169582Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:20:52.807177492Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:20:52.807182992Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:20:52.807188202Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:20:52.807195083Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:20:52.807222163Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"ea9546ae-2db3-406c-947f-41f10aa6a2f5","remote_addr":"[fd00:1122:3344:101::c]:36377","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":242,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9a05049c-89e6-4819-b9f3-c7cf39a94b79\", \"content-length\": \"127\", \"date\": \"Fri, 01 Mar 2024 23:20:52 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"9a05049c-89e6-4819-b9f3-c7cf39a94b79\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:20:52.862168539Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:20:53.015335421Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:36130"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:20:53.015402713Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"1b4eefc7-566f-458a-82db-52cfa24241c2","remote_addr":"[fd00:1122:3344:101::c]:36130","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":41,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:20:53.016182462Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:20:53.016201573Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:20:53.016470169Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:20:53.016655644Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"13048da2-80d8-4f9d-8cc6-0483c91e1be1","remote_addr":"[fd00:1122:3344:101::c]:36130","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":1124,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:21:04.756639828Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:64435"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:21:04.756695989Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"2a563122-2654-41ef-b5ed-b5b78ae369c5","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:64435","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":33,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:21:08.070197058Z","hostname":"buskin","pid":1671,"uri":"/metrics/collect/cc74b0a2-490e-4424-adad-bb6a277ade47","method":"GET","req_id":"7b45d95d-701a-4b16-b764-44c8f2dcfd4a","remote_addr":"[fd00:1122:3344:101::d]:60361","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":830,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:21:25.99872226Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:48762"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:21:25.998866373Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3af2d634-ca96-4144-9913-082ef495b783, kind: Service { id: 0a603215-e206-4aa0-8930-250f278a0d89 }, name: Name(\"external-dns-0a603215-e206-4aa0-8930-250f278a0d89\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 224, 81])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: d086f287-39ab-42aa-b1b5-4d91b0388531, kind: Service { id: 3f949516-3930-421a-bae2-b58ff4ea0dc8 }, name: Name(\"external-dns-3f949516-3930-421a-bae2-b58ff4ea0dc8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 146])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 06ae0dac-13b1-4ef0-995e-b56e4a7d3138, kind: Service { id: 933c0d3a-d49d-431b-8f65-51222f67e8f3 }, name: Name(\"nexus-933c0d3a-d49d-431b-8f65-51222f67e8f3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 46])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 9eeb19bd-897d-4d56-ae7e-bd0911ac9557, kind: Service { id: 264d5dd4-3eb2-4adb-b639-f0a26ab5534d }, name: Name(\"nexus-264d5dd4-3eb2-4adb-b639-f0a26ab5534d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 168, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: c6ce2abe-65af-4c28-bdee-3dca5bb68c72, kind: Service { id: 4ea96463-fca7-4b4a-8a70-4e88cc368897 }, name: Name(\"nexus-4ea96463-fca7-4b4a-8a70-4e88cc368897\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:21:25.998908985Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:21:25.998936865Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:21:25.998943886Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:21:25.998949666Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:21:25.998955456Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:21:25.998963886Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:21:25.998989067Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"2ec6c419-2a03-4264-896c-e0ed5e57229d","remote_addr":"[fd00:1122:3344:101::c]:48762","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":242,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5a7cc7ab-a701-4801-940f-4736dc19f29c\", \"content-length\": \"127\", \"date\": \"Fri, 01 Mar 2024 23:21:26 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"5a7cc7ab-a701-4801-940f-4736dc19f29c\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:21:26.049856403Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:21:26.138501184Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:45144"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:21:26.138595447Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"1fa3c52f-da0d-42f9-80cb-0285c28c238e","remote_addr":"[fd00:1122:3344:101::c]:45144","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":42,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:21:26.138804222Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:21:26.138839673Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:21:26.139098679Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:21:26.139309304Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"e75e4ab5-d585-43da-ad48-d9a300b313c1","remote_addr":"[fd00:1122:3344:101::c]:45144","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":536,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:21:34.791250706Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:58448"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:21:34.791309297Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"d47bd039-a539-40f2-ada6-b6f333fe0328","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:58448","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":30,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:21:38.070868304Z","hostname":"buskin","pid":1671,"uri":"/metrics/collect/cc74b0a2-490e-4424-adad-bb6a277ade47","method":"GET","req_id":"3155f456-2dad-472d-9c88-37b77f996872","remote_addr":"[fd00:1122:3344:101::d]:60361","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":1330,"response_code":"200"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:22:01.005815003Z","hostname":"buskin","pid":1671,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:776"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:22:02.954891549Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:40845"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:22:02.955669668Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3af2d634-ca96-4144-9913-082ef495b783, kind: Service { id: 0a603215-e206-4aa0-8930-250f278a0d89 }, name: Name(\"external-dns-0a603215-e206-4aa0-8930-250f278a0d89\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 224, 81])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: d086f287-39ab-42aa-b1b5-4d91b0388531, kind: Service { id: 3f949516-3930-421a-bae2-b58ff4ea0dc8 }, name: Name(\"external-dns-3f949516-3930-421a-bae2-b58ff4ea0dc8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 146])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 06ae0dac-13b1-4ef0-995e-b56e4a7d3138, kind: Service { id: 933c0d3a-d49d-431b-8f65-51222f67e8f3 }, name: Name(\"nexus-933c0d3a-d49d-431b-8f65-51222f67e8f3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 46])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 9eeb19bd-897d-4d56-ae7e-bd0911ac9557, kind: Service { id: 264d5dd4-3eb2-4adb-b639-f0a26ab5534d }, name: Name(\"nexus-264d5dd4-3eb2-4adb-b639-f0a26ab5534d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 168, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: c6ce2abe-65af-4c28-bdee-3dca5bb68c72, kind: Service { id: 4ea96463-fca7-4b4a-8a70-4e88cc368897 }, name: Name(\"nexus-4ea96463-fca7-4b4a-8a70-4e88cc368897\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:22:02.955711589Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:22:02.955742779Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:22:02.95574863Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:22:02.9557548Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:22:02.95576409Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:22:02.95577239Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:22:02.955798791Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"6ebff97e-3da9-4018-8334-0da1f3131854","remote_addr":"[fd00:1122:3344:101::c]:40845","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":882,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2b243ecc-bf46-4c57-8b2f-f8ef081d8b8b\", \"content-length\": \"127\", \"date\": \"Fri, 01 Mar 2024 23:22:02 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"2b243ecc-bf46-4c57-8b2f-f8ef081d8b8b\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:22:03.018919867Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:22:03.116411006Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:60089"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:22:03.116499948Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"7a478425-6ce4-4063-86d6-574c6a0f9a74","remote_addr":"[fd00:1122:3344:101::c]:60089","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":41,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:22:03.117498502Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:22:03.117532103Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:22:03.117795819Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:22:03.117992454Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"f9268c88-06be-4481-944f-71149306b6ad","remote_addr":"[fd00:1122:3344:101::c]:60089","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":1306,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:22:04.825103301Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:34836"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:22:04.825153132Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"06f2f40c-5b29-4b1d-aff7-3b9727191126","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:34836","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":35,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:22:08.069147588Z","hostname":"buskin","pid":1671,"uri":"/metrics/collect/cc74b0a2-490e-4424-adad-bb6a277ade47","method":"GET","req_id":"83b5bbf5-79ff-4b64-8466-9a77b8de0d85","remote_addr":"[fd00:1122:3344:101::d]:60361","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":186,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:22:34.863597813Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:47199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:22:34.863676065Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"660485ce-46da-4c09-b0af-3980e0a48b08","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:47199","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":50,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:22:38.069335073Z","hostname":"buskin","pid":1671,"uri":"/metrics/collect/cc74b0a2-490e-4424-adad-bb6a277ade47","method":"GET","req_id":"6e2abb09-13c8-4e27-828e-bcfcc1198931","remote_addr":"[fd00:1122:3344:101::d]:60361","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":189,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:23:04.896967723Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:33844"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:23:04.897050045Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"e6e29019-6efc-40a1-a210-8c7d573632e7","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:33844","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":50,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:23:08.069485399Z","hostname":"buskin","pid":1671,"uri":"/metrics/collect/cc74b0a2-490e-4424-adad-bb6a277ade47","method":"GET","req_id":"645e311b-fdf2-423f-b81b-09c3d706c601","remote_addr":"[fd00:1122:3344:101::d]:60361","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":747,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:23:12.873091931Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:53181"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:23:12.873266985Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3af2d634-ca96-4144-9913-082ef495b783, kind: Service { id: 0a603215-e206-4aa0-8930-250f278a0d89 }, name: Name(\"external-dns-0a603215-e206-4aa0-8930-250f278a0d89\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 224, 81])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: d086f287-39ab-42aa-b1b5-4d91b0388531, kind: Service { id: 3f949516-3930-421a-bae2-b58ff4ea0dc8 }, name: Name(\"external-dns-3f949516-3930-421a-bae2-b58ff4ea0dc8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 146])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 06ae0dac-13b1-4ef0-995e-b56e4a7d3138, kind: Service { id: 933c0d3a-d49d-431b-8f65-51222f67e8f3 }, name: Name(\"nexus-933c0d3a-d49d-431b-8f65-51222f67e8f3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 46])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 9eeb19bd-897d-4d56-ae7e-bd0911ac9557, kind: Service { id: 264d5dd4-3eb2-4adb-b639-f0a26ab5534d }, name: Name(\"nexus-264d5dd4-3eb2-4adb-b639-f0a26ab5534d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 168, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: c6ce2abe-65af-4c28-bdee-3dca5bb68c72, kind: Service { id: 4ea96463-fca7-4b4a-8a70-4e88cc368897 }, name: Name(\"nexus-4ea96463-fca7-4b4a-8a70-4e88cc368897\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:23:12.873303866Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:23:12.873323556Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:23:12.873329986Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:23:12.873335697Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:23:12.873343517Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:23:12.873350527Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:23:12.873375728Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"39a28dfd-3bb5-450c-bb0f-ee1aa8fcaad1","remote_addr":"[fd00:1122:3344:101::c]:53181","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":218,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"fe9b1487-d1b4-479a-99ab-da9c8307c225\", \"content-length\": \"127\", \"date\": \"Fri, 01 Mar 2024 23:23:12 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"fe9b1487-d1b4-479a-99ab-da9c8307c225\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:23:12.93183045Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:23:13.027105864Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:41591"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:23:13.027191916Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"aa7e92c3-8387-4ab7-8a82-1dff384f5bd3","remote_addr":"[fd00:1122:3344:101::c]:41591","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":52,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:23:13.027404741Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:23:13.027441972Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:23:13.027751569Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:23:13.027970765Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"4710c32e-f577-4222-81ab-6719dea5d11c","remote_addr":"[fd00:1122:3344:101::c]:41591","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":595,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:23:34.936226377Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:56457"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:23:34.936297039Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"94e4af5c-f69b-4684-875f-6f9c84df384b","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:56457","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":45,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:23:38.068460765Z","hostname":"buskin","pid":1671,"uri":"/metrics/collect/cc74b0a2-490e-4424-adad-bb6a277ade47","method":"GET","req_id":"5f75d124-5f1a-42c1-9660-160cd8094fb5","remote_addr":"[fd00:1122:3344:101::d]:60361","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":186,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:24:04.969095386Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:39213"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:24:04.969130567Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"00b2cd18-169e-4f8a-ac4b-d8b069910a22","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:39213","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":27,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:24:08.069567796Z","hostname":"buskin","pid":1671,"uri":"/metrics/collect/cc74b0a2-490e-4424-adad-bb6a277ade47","method":"GET","req_id":"a1b23788-4441-4bc5-a050-f365e6ebc466","remote_addr":"[fd00:1122:3344:101::d]:60361","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":952,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:24:09.801225264Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:44466"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:24:09.801355757Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3af2d634-ca96-4144-9913-082ef495b783, kind: Service { id: 0a603215-e206-4aa0-8930-250f278a0d89 }, name: Name(\"external-dns-0a603215-e206-4aa0-8930-250f278a0d89\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 224, 81])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: d086f287-39ab-42aa-b1b5-4d91b0388531, kind: Service { id: 3f949516-3930-421a-bae2-b58ff4ea0dc8 }, name: Name(\"external-dns-3f949516-3930-421a-bae2-b58ff4ea0dc8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 146])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 06ae0dac-13b1-4ef0-995e-b56e4a7d3138, kind: Service { id: 933c0d3a-d49d-431b-8f65-51222f67e8f3 }, name: Name(\"nexus-933c0d3a-d49d-431b-8f65-51222f67e8f3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 46])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 9eeb19bd-897d-4d56-ae7e-bd0911ac9557, kind: Service { id: 264d5dd4-3eb2-4adb-b639-f0a26ab5534d }, name: Name(\"nexus-264d5dd4-3eb2-4adb-b639-f0a26ab5534d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 168, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: c6ce2abe-65af-4c28-bdee-3dca5bb68c72, kind: Service { id: 4ea96463-fca7-4b4a-8a70-4e88cc368897 }, name: Name(\"nexus-4ea96463-fca7-4b4a-8a70-4e88cc368897\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:24:09.801398278Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:24:09.801413639Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:24:09.801427979Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:24:09.801435879Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:24:09.801441269Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:24:09.801446709Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:24:09.80149152Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"5a784cc7-1a40-4505-9db6-318d5247efa1","remote_addr":"[fd00:1122:3344:101::c]:44466","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":241,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b50cd092-5bcb-4c64-b6f0-56672fa14452\", \"content-length\": \"127\", \"date\": \"Fri, 01 Mar 2024 23:24:09 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"b50cd092-5bcb-4c64-b6f0-56672fa14452\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:24:09.852696025Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:24:09.941000608Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:41823"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:24:09.94107392Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"b6e6ef9c-0031-4c62-9cc9-09f0f9a4d332","remote_addr":"[fd00:1122:3344:101::c]:41823","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":34,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:24:09.941706515Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:24:09.941736196Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:24:09.941967382Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:24:09.942135236Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"d427287b-5c16-4834-b07e-23155e88f6ff","remote_addr":"[fd00:1122:3344:101::c]:41823","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":919,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:24:35.001583677Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:56785"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:24:35.001647109Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"e9409a5f-2f7a-43ab-b587-acb1a872c39b","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:56785","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":33,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:24:38.069365696Z","hostname":"buskin","pid":1671,"uri":"/metrics/collect/cc74b0a2-490e-4424-adad-bb6a277ade47","method":"GET","req_id":"935dbef4-e4f9-4b71-a66d-2aba71a0e248","remote_addr":"[fd00:1122:3344:101::d]:60361","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":210,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:24:44.155147046Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:35630"} {"msg":"ensuring instance is registered","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:24:44.169105748Z","hostname":"buskin","pid":1671,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:586","propolis_addr":"[fd00:1122:3344:101::1:0]:12400","vmm_runtime":"VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-03-01T23:24:43.763784Z }","instance_runtime":"InstanceRuntimeState { propolis_id: Some(1dbdb96d-b477-45b3-abdd-75107e97f19d), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-03-01T23:24:43.801001870Z }","hardware":"InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: 73cd6c2b-f9ce-4b74-bb8a-b10977417987, kind: Instance { id: 6ad225a2-e890-47c4-a0fe-65377d224656 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 246, 110, 10])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(14634911), primary: true, slot: 0 }], source_nat: SourceNatConfig { ip: 10.151.1.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.1.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 73cd6c2b-f9ce-4b74-bb8a-b10977417987, kind: Instance { id: 6ad225a2-e890-47c4-a0fe-65377d224656 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 246, 110, 10])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(14634911), primary: true, slot: 0 }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 73cd6c2b-f9ce-4b74-bb8a-b10977417987, kind: Instance { id: 6ad225a2-e890-47c4-a0fe-65377d224656 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 246, 110, 10])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(14634911), primary: true, slot: 0 }], filter_hosts: Some([Vpc(Vni(14634911))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 73cd6c2b-f9ce-4b74-bb8a-b10977417987, kind: Instance { id: 6ad225a2-e890-47c4-a0fe-65377d224656 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 246, 110, 10])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(14634911), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-138c908c618\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: de6b4238-e605-4fef-8afb-a3b1b5732907, read_only_parent: Some(Volume { block_size: 512, id: 73d568bd-157c-4e4c-bdca-fc064621a7ec, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 3ec606f5-51aa-4519-86fd-365fea9e2e4b, key: Some(\"SrVZm/danurkCAZJX4ipTIwMKHmfsTfpdC6z+TTNHuM=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::18]:19001\", \"[fd00:1122:3344:101::13]:19002\", \"[fd00:1122:3344:101::16]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: de6b4238-e605-4fef-8afb-a3b1b5732907, key: Some(\"X1Efk+zAbzRE1CILfzJuhcw8DRAtf3qu3Ze33sGmeCg=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::18]:19002\", \"[fd00:1122:3344:101::19]:19000\", \"[fd00:1122:3344:101::12]:19000\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAAALFrthWGFYAAAWu2FYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAADBa7YVhhWAAAFrthWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiI2YWQyMjVhMi1lODkwLTQ3YzQtYTBmZS02NTM3N2QyMjQ2NTYiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJSEZMMllBN0FaYWFSR2FwSkp3aWJpbDJEenhtMllFTExrdkRBeENEcEhVbiJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }","propolis_id":"1dbdb96d-b477-45b3-abdd-75107e97f19d","instance_id":"6ad225a2-e890-47c4-a0fe-65377d224656"} {"msg":"registering new instance","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:24:44.16919023Z","hostname":"buskin","pid":1671,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:619","instance_id":"6ad225a2-e890-47c4-a0fe-65377d224656"} {"msg":"initializing new Instance","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:24:44.171138178Z","hostname":"buskin","pid":1671,"component":"InstanceManager","file":"sled-agent/src/instance.rs:932","state":"InstanceInitialState { hardware: InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: 73cd6c2b-f9ce-4b74-bb8a-b10977417987, kind: Instance { id: 6ad225a2-e890-47c4-a0fe-65377d224656 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 246, 110, 10])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(14634911), primary: true, slot: 0 }], source_nat: SourceNatConfig { ip: 10.151.1.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.1.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 73cd6c2b-f9ce-4b74-bb8a-b10977417987, kind: Instance { id: 6ad225a2-e890-47c4-a0fe-65377d224656 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 246, 110, 10])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(14634911), primary: true, slot: 0 }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 73cd6c2b-f9ce-4b74-bb8a-b10977417987, kind: Instance { id: 6ad225a2-e890-47c4-a0fe-65377d224656 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 246, 110, 10])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(14634911), primary: true, slot: 0 }], filter_hosts: Some([Vpc(Vni(14634911))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 73cd6c2b-f9ce-4b74-bb8a-b10977417987, kind: Instance { id: 6ad225a2-e890-47c4-a0fe-65377d224656 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 246, 110, 10])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(14634911), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-138c908c618\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: de6b4238-e605-4fef-8afb-a3b1b5732907, read_only_parent: Some(Volume { block_size: 512, id: 73d568bd-157c-4e4c-bdca-fc064621a7ec, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 3ec606f5-51aa-4519-86fd-365fea9e2e4b, key: Some(\"SrVZm/danurkCAZJX4ipTIwMKHmfsTfpdC6z+TTNHuM=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::18]:19001\", \"[fd00:1122:3344:101::13]:19002\", \"[fd00:1122:3344:101::16]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: de6b4238-e605-4fef-8afb-a3b1b5732907, key: Some(\"X1Efk+zAbzRE1CILfzJuhcw8DRAtf3qu3Ze33sGmeCg=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::18]:19002\", \"[fd00:1122:3344:101::19]:19000\", \"[fd00:1122:3344:101::12]:19000\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAAALFrthWGFYAAAWu2FYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAADBa7YVhhWAAAFrthWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiI2YWQyMjVhMi1lODkwLTQ3YzQtYTBmZS02NTM3N2QyMjQ2NTYiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJSEZMMllBN0FaYWFSR2FwSkp3aWJpbDJEenhtMllFTExrdkRBeENEcEhVbiJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }, instance_runtime: InstanceRuntimeState { propolis_id: Some(1dbdb96d-b477-45b3-abdd-75107e97f19d), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-03-01T23:24:43.801001870Z }, vmm_runtime: VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-03-01T23:24:43.763784Z }, propolis_addr: [fd00:1122:3344:101::1:0]:12400 }","propolis_id":"1dbdb96d-b477-45b3-abdd-75107e97f19d","instance_id":"6ad225a2-e890-47c4-a0fe-65377d224656"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:24:44.171943907Z","hostname":"buskin","pid":1671,"uri":"/instances/6ad225a2-e890-47c4-a0fe-65377d224656","method":"PUT","req_id":"6ae60ebf-7682-4052-8dc1-00c02920f568","remote_addr":"[fd00:1122:3344:101::a]:35630","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":16771,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:24:44.291467125Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:61590"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:24:44.319344958Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte6\", ip: 172.30.0.5, mac: MacAddr6([168, 64, 37, 246, 110, 10]), slot: 0, vni: Vni { inner: 14634911 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.0.1 }, vnic: \"vopte6\" } }"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:24:44.319486862Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Configuring new Omicron zone: oxz_propolis-server_1dbdb96d-b477-45b3-abdd-75107e97f19d","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:24:44.353902445Z","hostname":"buskin","pid":1671,"instance_id":"6ad225a2-e890-47c4-a0fe-65377d224656","component":"InstanceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_propolis-server_1dbdb96d-b477-45b3-abdd-75107e97f19d","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:24:44.374600372Z","hostname":"buskin","pid":1671,"instance_id":"6ad225a2-e890-47c4-a0fe-65377d224656","component":"InstanceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_propolis-server_1dbdb96d-b477-45b3-abdd-75107e97f19d:\n\n\n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:24:46.426806793Z","hostname":"buskin","pid":1671,"instance_id":"6ad225a2-e890-47c4-a0fe-65377d224656","component":"InstanceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:24:46.427026189Z","hostname":"buskin","pid":1671,"zone":"oxz_propolis-server_1dbdb96d-b477-45b3-abdd-75107e97f19d","instance_id":"6ad225a2-e890-47c4-a0fe-65377d224656","component":"InstanceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_propolis-server_1dbdb96d-b477-45b3-abdd-75107e97f19d\") failed: Property not found. retry in 68.62818ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:24:46.639548944Z","hostname":"buskin","pid":1671,"zone":"oxz_propolis-server_1dbdb96d-b477-45b3-abdd-75107e97f19d","instance_id":"6ad225a2-e890-47c4-a0fe-65377d224656","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_1dbdb96d-b477-45b3-abdd-75107e97f19d\") failed: Property not found. retry in 123.033942ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:24:46.72634561Z","hostname":"buskin","pid":1671,"zone":"oxz_propolis-server_1dbdb96d-b477-45b3-abdd-75107e97f19d","instance_id":"6ad225a2-e890-47c4-a0fe-65377d224656","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_1dbdb96d-b477-45b3-abdd-75107e97f19d\") failed: Property not found. retry in 248.561806ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:24:46.869065266Z","hostname":"buskin","pid":1671,"zone":"oxz_propolis-server_1dbdb96d-b477-45b3-abdd-75107e97f19d","instance_id":"6ad225a2-e890-47c4-a0fe-65377d224656","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_1dbdb96d-b477-45b3-abdd-75107e97f19d\") failed: Property not found. retry in 237.953928ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:24:47.132441898Z","hostname":"buskin","pid":1671,"zone":"oxz_propolis-server_1dbdb96d-b477-45b3-abdd-75107e97f19d","instance_id":"6ad225a2-e890-47c4-a0fe-65377d224656","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_1dbdb96d-b477-45b3-abdd-75107e97f19d\") failed: Property not found. retry in 491.57372ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:24:47.384872532Z","hostname":"buskin","pid":1671,"zone":"oxz_propolis-server_1dbdb96d-b477-45b3-abdd-75107e97f19d","instance_id":"6ad225a2-e890-47c4-a0fe-65377d224656","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_1dbdb96d-b477-45b3-abdd-75107e97f19d\") failed: Property not found. retry in 1.264722371s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:24:47.887506354Z","hostname":"buskin","pid":1671,"zone":"oxz_propolis-server_1dbdb96d-b477-45b3-abdd-75107e97f19d","instance_id":"6ad225a2-e890-47c4-a0fe-65377d224656","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_1dbdb96d-b477-45b3-abdd-75107e97f19d\") failed: Property not found. retry in 1.163069994s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:24:49.162551948Z","hostname":"buskin","pid":1671,"zone":"oxz_propolis-server_1dbdb96d-b477-45b3-abdd-75107e97f19d","instance_id":"6ad225a2-e890-47c4-a0fe-65377d224656","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_1dbdb96d-b477-45b3-abdd-75107e97f19d\") failed: Property not found. retry in 548.271981ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:24:50.335656694Z","hostname":"buskin","pid":1671,"zone":"oxz_propolis-server_1dbdb96d-b477-45b3-abdd-75107e97f19d","instance_id":"6ad225a2-e890-47c4-a0fe-65377d224656","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_1dbdb96d-b477-45b3-abdd-75107e97f19d\") failed: Property not found. retry in 1.368619568s","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:24:50.895803465Z","hostname":"buskin","pid":1671,"zone":"oxz_propolis-server_1dbdb96d-b477-45b3-abdd-75107e97f19d","instance_id":"6ad225a2-e890-47c4-a0fe-65377d224656","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:24:51.317557687Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:36776"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:24:51.317827854Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3af2d634-ca96-4144-9913-082ef495b783, kind: Service { id: 0a603215-e206-4aa0-8930-250f278a0d89 }, name: Name(\"external-dns-0a603215-e206-4aa0-8930-250f278a0d89\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 224, 81])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: d086f287-39ab-42aa-b1b5-4d91b0388531, kind: Service { id: 3f949516-3930-421a-bae2-b58ff4ea0dc8 }, name: Name(\"external-dns-3f949516-3930-421a-bae2-b58ff4ea0dc8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 146])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 06ae0dac-13b1-4ef0-995e-b56e4a7d3138, kind: Service { id: 933c0d3a-d49d-431b-8f65-51222f67e8f3 }, name: Name(\"nexus-933c0d3a-d49d-431b-8f65-51222f67e8f3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 46])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 9eeb19bd-897d-4d56-ae7e-bd0911ac9557, kind: Service { id: 264d5dd4-3eb2-4adb-b639-f0a26ab5534d }, name: Name(\"nexus-264d5dd4-3eb2-4adb-b639-f0a26ab5534d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 168, 94])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: c6ce2abe-65af-4c28-bdee-3dca5bb68c72, kind: Service { id: 4ea96463-fca7-4b4a-8a70-4e88cc368897 }, name: Name(\"nexus-4ea96463-fca7-4b4a-8a70-4e88cc368897\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 195, 22])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:24:51.317883805Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:24:51.317924706Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:24:51.317952606Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:24:51.317969607Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:24:51.318003908Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:24:51.318039799Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:24:51.318054869Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"cd13a878-c68b-4111-a4ea-dd35f7abffcc","remote_addr":"[fd00:1122:3344:101::c]:36776","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":421,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a9ddaf63-0068-4cdd-be01-c8517e8f4dba\", \"content-length\": \"127\", \"date\": \"Fri, 01 Mar 2024 23:24:51 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"a9ddaf63-0068-4cdd-be01-c8517e8f4dba\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:24:51.408487694Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:24:51.581763418Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:38176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:24:51.582514666Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"6276b270-c31b-4ab9-a81f-98398a1a7a93","remote_addr":"[fd00:1122:3344:101::c]:38176","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":585,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:24:51.582749042Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:24:51.582788703Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:24:51.583129352Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:24:51.583401668Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"1027aabf-9cce-4cbf-b5fe-c349f7790379","remote_addr":"[fd00:1122:3344:101::c]:38176","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":724,"response_code":"200"} {"msg":"Started propolis in zone: oxz_propolis-server_1dbdb96d-b477-45b3-abdd-75107e97f19d","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:24:52.292078899Z","hostname":"buskin","pid":1671,"instance_id":"6ad225a2-e890-47c4-a0fe-65377d224656","component":"InstanceManager","file":"sled-agent/src/instance.rs:1416"} {"msg":"wait for service Some(\"oxz_propolis-server_1dbdb96d-b477-45b3-abdd-75107e97f19d\") failed: Property not found. retry in 46.98504ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:24:52.301021428Z","hostname":"buskin","pid":1671,"instance_id":"6ad225a2-e890-47c4-a0fe-65377d224656","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_1dbdb96d-b477-45b3-abdd-75107e97f19d\") failed: Property not found. retry in 108.61635ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:24:52.35703351Z","hostname":"buskin","pid":1671,"instance_id":"6ad225a2-e890-47c4-a0fe-65377d224656","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_1dbdb96d-b477-45b3-abdd-75107e97f19d\") failed: Property not found. retry in 247.741297ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:24:52.473301927Z","hostname":"buskin","pid":1671,"instance_id":"6ad225a2-e890-47c4-a0fe-65377d224656","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Propolis SMF service is online","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:24:52.732154388Z","hostname":"buskin","pid":1671,"instance_id":"6ad225a2-e890-47c4-a0fe-65377d224656","component":"InstanceManager","file":"sled-agent/src/instance.rs:1425"} {"msg":"Propolis HTTP server online","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:24:52.766800157Z","hostname":"buskin","pid":1671,"instance_id":"6ad225a2-e890-47c4-a0fe-65377d224656","component":"InstanceManager","file":"sled-agent/src/instance.rs:1439"} {"msg":"Sending ensure request to propolis: InstanceEnsureRequest { cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAAALFrthWGFYAAAWu2FYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAADBa7YVhhWAAAFrthWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiI2YWQyMjVhMi1lODkwLTQ3YzQtYTBmZS02NTM3N2QyMjQ2NTYiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJSEZMMllBN0FaYWFSR2FwSkp3aWJpbDJEenhtMllFTExrdkRBeENEcEhVbiJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\"), disks: [DiskRequest { device: \"nvme\", name: \"disk-138c908c618\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: de6b4238-e605-4fef-8afb-a3b1b5732907, read_only_parent: Some(Volume { block_size: 512, id: 73d568bd-157c-4e4c-bdca-fc064621a7ec, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 3ec606f5-51aa-4519-86fd-365fea9e2e4b, key: Some(\"SrVZm/danurkCAZJX4ipTIwMKHmfsTfpdC6z+TTNHuM=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::18]:19001\", \"[fd00:1122:3344:101::13]:19002\", \"[fd00:1122:3344:101::16]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: de6b4238-e605-4fef-8afb-a3b1b5732907, key: Some(\"X1Efk+zAbzRE1CILfzJuhcw8DRAtf3qu3Ze33sGmeCg=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::18]:19002\", \"[fd00:1122:3344:101::19]:19000\", \"[fd00:1122:3344:101::12]:19000\"] } }] } }], migrate: None, nics: [NetworkInterfaceRequest { name: \"vopte6\", slot: Slot(0) }], properties: InstanceProperties { bootrom_id: 00000000-0000-0000-0000-000000000000, description: \"Test description\", id: 6ad225a2-e890-47c4-a0fe-65377d224656, image_id: 00000000-0000-0000-0000-000000000000, memory: 1024, name: \"localshark\", vcpus: 2 } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:24:52.770556059Z","hostname":"buskin","pid":1671,"instance_id":"6ad225a2-e890-47c4-a0fe-65377d224656","component":"InstanceManager","file":"sled-agent/src/instance.rs:681"} {"msg":"result of instance_ensure call is Ok(InstanceEnsureResponse { migrate: None })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:24:52.926366896Z","hostname":"buskin","pid":1671,"instance_id":"6ad225a2-e890-47c4-a0fe-65377d224656","component":"InstanceManager","file":"sled-agent/src/instance.rs:683"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:24:52.926799556Z","hostname":"buskin","pid":1671,"uri":"/instances/6ad225a2-e890-47c4-a0fe-65377d224656/state","method":"PUT","req_id":"a7f1fe2c-4293-42e2-94a1-d309f1c725e6","remote_addr":"[fd00:1122:3344:101::a]:61590","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":8635241,"response_code":"200"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Starting), migration_status: NoMigration, time: 2024-03-01T23:24:52.927087043Z }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:24:52.927115064Z","hostname":"buskin","pid":1671,"instance_id":"6ad225a2-e890-47c4-a0fe-65377d224656","component":"InstanceManager","file":"sled-agent/src/instance.rs:565"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:24:52.927132524Z","hostname":"buskin","pid":1671,"instance_id":"6ad225a2-e890-47c4-a0fe-65377d224656","component":"InstanceManager","file":"sled-agent/src/instance.rs:587","new_vmm_state":"VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-03-01T23:24:52.927087043Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(1dbdb96d-b477-45b3-abdd-75107e97f19d), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-03-01T23:24:43.801001870Z }","propolis_id":"1dbdb96d-b477-45b3-abdd-75107e97f19d"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:24:52.927146695Z","hostname":"buskin","pid":1671,"instance_id":"6ad225a2-e890-47c4-a0fe-65377d224656","component":"InstanceManager","file":"sled-agent/src/instance.rs:487","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(1dbdb96d-b477-45b3-abdd-75107e97f19d), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-03-01T23:24:43.801001870Z }, propolis_id: 1dbdb96d-b477-45b3-abdd-75107e97f19d, vmm_state: VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-03-01T23:24:52.927087043Z } }","instance_id":"6ad225a2-e890-47c4-a0fe-65377d224656"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Running), migration_status: NoMigration, time: 2024-03-01T23:24:52.985886354Z }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:24:52.985906194Z","hostname":"buskin","pid":1671,"instance_id":"6ad225a2-e890-47c4-a0fe-65377d224656","component":"InstanceManager","file":"sled-agent/src/instance.rs:565"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:24:52.985928555Z","hostname":"buskin","pid":1671,"instance_id":"6ad225a2-e890-47c4-a0fe-65377d224656","component":"InstanceManager","file":"sled-agent/src/instance.rs:587","new_vmm_state":"VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-03-01T23:24:52.985886354Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(1dbdb96d-b477-45b3-abdd-75107e97f19d), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-03-01T23:24:43.801001870Z }","propolis_id":"1dbdb96d-b477-45b3-abdd-75107e97f19d"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:24:52.985939805Z","hostname":"buskin","pid":1671,"instance_id":"6ad225a2-e890-47c4-a0fe-65377d224656","component":"InstanceManager","file":"sled-agent/src/instance.rs:487","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(1dbdb96d-b477-45b3-abdd-75107e97f19d), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-03-01T23:24:43.801001870Z }, propolis_id: 1dbdb96d-b477-45b3-abdd-75107e97f19d, vmm_state: VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-03-01T23:24:52.985886354Z } }","instance_id":"6ad225a2-e890-47c4-a0fe-65377d224656"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:25:05.03372718Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:34966"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:25:05.033837752Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"5e24d316-d1d3-4c4d-88ce-ea1d01d48679","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:34966","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":44,"response_code":"200"} {"msg":"running automatic periodic zone bundle cleanup","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:25:06.929479447Z","hostname":"buskin","pid":1671,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:299"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:25:06.929517619Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:25:06.982703841Z","hostname":"buskin","pid":1671,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:288","time_to_next_cleanup":"599.99999985s","next_cleanup":"Instant { tv_sec: 2225, tv_nsec: 331017932 }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:25:08.070107098Z","hostname":"buskin","pid":1671,"uri":"/metrics/collect/cc74b0a2-490e-4424-adad-bb6a277ade47","method":"GET","req_id":"cfe8f634-583c-4e1d-903d-7d7a87cc4202","remote_addr":"[fd00:1122:3344:101::d]:60361","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":1107,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:25:13.245048055Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:47381"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:25:13.246157573Z","hostname":"buskin","pid":1671,"uri":"/instances/6ad225a2-e890-47c4-a0fe-65377d224656/state","method":"PUT","req_id":"a3f0550b-c0f8-4db1-9a7d-966d0c40e554","remote_addr":"[fd00:1122:3344:101::a]:47381","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":1073,"response_code":"200"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Stopping), migration_status: NoMigration, time: 2024-03-01T23:25:13.246251035Z }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:25:13.246279636Z","hostname":"buskin","pid":1671,"instance_id":"6ad225a2-e890-47c4-a0fe-65377d224656","component":"InstanceManager","file":"sled-agent/src/instance.rs:565"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:25:13.246292336Z","hostname":"buskin","pid":1671,"instance_id":"6ad225a2-e890-47c4-a0fe-65377d224656","component":"InstanceManager","file":"sled-agent/src/instance.rs:587","new_vmm_state":"VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-03-01T23:25:13.246251035Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(1dbdb96d-b477-45b3-abdd-75107e97f19d), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-03-01T23:24:43.801001870Z }","propolis_id":"1dbdb96d-b477-45b3-abdd-75107e97f19d"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:25:13.246319366Z","hostname":"buskin","pid":1671,"instance_id":"6ad225a2-e890-47c4-a0fe-65377d224656","component":"InstanceManager","file":"sled-agent/src/instance.rs:487","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(1dbdb96d-b477-45b3-abdd-75107e97f19d), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-03-01T23:24:43.801001870Z }, propolis_id: 1dbdb96d-b477-45b3-abdd-75107e97f19d, vmm_state: VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-03-01T23:25:13.246251035Z } }","instance_id":"6ad225a2-e890-47c4-a0fe-65377d224656"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Destroyed), migration_status: NoMigration, time: 2024-03-01T23:25:14.042879219Z }","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:25:14.0429089Z","hostname":"buskin","pid":1671,"instance_id":"6ad225a2-e890-47c4-a0fe-65377d224656","component":"InstanceManager","file":"sled-agent/src/instance.rs:565"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:25:14.042945201Z","hostname":"buskin","pid":1671,"instance_id":"6ad225a2-e890-47c4-a0fe-65377d224656","component":"InstanceManager","file":"sled-agent/src/instance.rs:587","new_vmm_state":"VmmRuntimeState { state: Destroyed, gen: Generation(6), time_updated: 2024-03-01T23:25:14.042879219Z }","new_instance_state":"InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-03-01T23:25:14.042879219Z }","propolis_id":"1dbdb96d-b477-45b3-abdd-75107e97f19d"} {"msg":"terminating VMM that has exited","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:25:14.042966091Z","hostname":"buskin","pid":1671,"instance_id":"6ad225a2-e890-47c4-a0fe-65377d224656","component":"InstanceManager","file":"sled-agent/src/instance.rs:602","instance_id":"6ad225a2-e890-47c4-a0fe-65377d224656"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:25:14.042974221Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:25:14.043118715Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"creating zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:25:14.043167946Z","hostname":"buskin","pid":1671,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:436","context":"ZoneBundleContext { storage_dirs: [\"/pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/debug/bundle/zone\", \"/pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/debug/bundle/zone\"], cause: TerminatedInstance, extra_log_dirs: [\"/pool/ext/14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/debug\", \"/pool/ext/24b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/debug\", \"/pool/ext/31bd71cd-4736-4a12-a387-9b74b050396f/crypt/debug\", \"/pool/ext/616b26df-e62a-4c68-b506-f4a923d8aaf7/crypt/debug\", \"/pool/ext/cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crypt/debug\", \"/pool/ext/ceb4461c-cf56-4719-ad3c-14430bfdfb60/crypt/debug\", \"/pool/ext/d462a7f7-b628-40fe-80ff-4e4189e2d62b/crypt/debug\", \"/pool/ext/e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/debug\", \"/pool/ext/f4b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/debug\"] }","zone_name":"oxz_propolis-server_1dbdb96d-b477-45b3-abdd-75107e97f19d"} {"msg":"finished zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:25:14.919998176Z","hostname":"buskin","pid":1671,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:1235","metadata":"ZoneBundleMetadata { id: ZoneBundleId { zone_name: \"oxz_propolis-server_1dbdb96d-b477-45b3-abdd-75107e97f19d\", bundle_id: 66c1c84b-a616-424a-bbf2-1f9424d6f9ff }, time_created: 2024-03-01T23:25:14.043349621Z, version: 0, cause: TerminatedInstance }"} {"msg":"Halting and removing zone: oxz_propolis-server_1dbdb96d-b477-45b3-abdd-75107e97f19d","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:25:14.920068207Z","hostname":"buskin","pid":1671,"instance_id":"6ad225a2-e890-47c4-a0fe-65377d224656","component":"InstanceManager","file":"sled-agent/src/instance.rs:776"} {"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:25:16.556217206Z","hostname":"buskin","pid":1671,"instance_id":"6ad225a2-e890-47c4-a0fe-65377d224656","component":"InstanceManager","file":"illumos-utils/src/zone.rs:269"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:25:16.582475479Z","hostname":"buskin","pid":1671,"instance_id":"6ad225a2-e890-47c4-a0fe-65377d224656","component":"InstanceManager","file":"sled-agent/src/instance.rs:487","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-03-01T23:25:14.042879219Z }, propolis_id: 1dbdb96d-b477-45b3-abdd-75107e97f19d, vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-03-01T23:25:16.582408447Z } }","instance_id":"6ad225a2-e890-47c4-a0fe-65377d224656"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:25:16.65641632Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:55966"} {"msg":"unset_virtual_nic_host unimplmented","v":0,"name":"SledAgent","level":40,"time":"2024-03-01T23:25:16.668770763Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:620"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:25:16.668796164Z","hostname":"buskin","pid":1671,"uri":"/v2p/73cd6c2b-f9ce-4b74-bb8a-b10977417987","method":"DELETE","req_id":"fc971821-2a92-4a82-9ed6-4b27f182c4f1","remote_addr":"[fd00:1122:3344:101::c]:55966","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":12224,"response_code":"204"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:25:17.155338913Z","hostname":"buskin","pid":1671,"instance_id":"6ad225a2-e890-47c4-a0fe-65377d224656","component":"InstanceManager","file":"sled-agent/src/instance.rs:487","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-03-01T23:25:14.042879219Z }, propolis_id: 1dbdb96d-b477-45b3-abdd-75107e97f19d, vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-03-01T23:25:16.582408447Z } }","instance_id":"6ad225a2-e890-47c4-a0fe-65377d224656"} {"msg":"State monitoring task complete","v":0,"name":"SledAgent","level":30,"time":"2024-03-01T23:25:17.158577022Z","hostname":"buskin","pid":1671,"instance_id":"6ad225a2-e890-47c4-a0fe-65377d224656","component":"InstanceManager","file":"sled-agent/src/instance.rs:719"}