[ Feb 15 05:55:26 Enabled. ] [ Feb 15 05:55:26 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ] [ Feb 15 05:55:26 Method "start" exited with status 0. ] [ Feb 15 05:55:26 Rereading configuration. ] [ Feb 15 05:55:26 No 'refresh' method defined. Treating as :true. ] note: configured to log to "/dev/stdout" {"msg":"finding links [\"net0\", \"net1\"]","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:26.405227362Z","hostname":"buskin","pid":1671,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:247"} {"msg":"Starting mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:26.456988843Z","hostname":"buskin","pid":1671,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:254","addr-objs":"[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]"} {"msg":"Importing mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:26.457019704Z","hostname":"buskin","pid":1671,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:42","path":"/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"} {"msg":"Setting mg-ddm interfaces","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:26.566277633Z","hostname":"buskin","pid":1671,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:50","interfaces":"(\"net0/ll\" \"net1/ll\")"} {"msg":"Enabling mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:26.575473725Z","hostname":"buskin","pid":1671,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:56"} {"msg":"Ensuring zfs key directory exists","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:26.583792938Z","hostname":"buskin","pid":1671,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:264","path":"/var/run/oxide/"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.188427608Z","hostname":"buskin","pid":1671,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:91","prefix":"Ipv6Prefix { addr: fdb0:18c0:4d0d:9fb2::, len: 64 }"} {"msg":"Starting KeyManager","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.329386789Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:117"} {"msg":"Starting StorageManager","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.32941795Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:129"} {"msg":"Starting StorageMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.329557704Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:141"} {"msg":"Starting HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.329696899Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:162","sled_mode":"Scrimlet { asic: SoftNpuZone }"} {"msg":"DumpSetup poll loop started.","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.329708409Z","hostname":"buskin","pid":1671,"component":"DumpSetup-archival","file":"sled-agent/src/dump_setup.rs:268"} {"msg":"Creating HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.32973155Z","hostname":"buskin","pid":1671,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:609"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.329772741Z","hostname":"buskin","pid":1671,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:776"} {"msg":"Generated i86pc baseboard Pc { identifier: \"buskin\", model: \"i86pc\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.332727388Z","hostname":"buskin","pid":1671,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:535"} {"msg":"Device is not a Gimlet (i86pc), proceeding with null hardware view","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:55:27.332739799Z","hostname":"buskin","pid":1671,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:658"} {"msg":"Starting HardwareMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.332745239Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:176"} {"msg":"Monitoring for hardware updates","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.332751129Z","hostname":"buskin","pid":1671,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:716"} {"msg":"Upserting synthetic zpool to Storage Manager: oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.332801951Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.332806341Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.332813111Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.332817571Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.332821021Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.332825882Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.332833372Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.332836842Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.332841012Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxp_31bd71cd-4736-4a12-a387-9b74b050396f","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.332844032Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Upserting synthetic zpool to Storage Manager: oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.332849262Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:232"} {"msg":"Waiting for boot disk","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.332852532Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:88"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.332856683Z","hostname":"buskin","pid":1671,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:241","underlay_network_info":"None"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: a462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal } }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.332898924Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Channel capacity decreased","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.516159653Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:326","current":"245","previous":"256"} {"msg":"Found boot disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.516209355Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:90"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: b462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal } }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.516217906Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.689198077Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.689247389Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.68926191Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.68926824Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.68927546Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.68928299Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.689295281Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.689301541Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received AddDisk(Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.689306811Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received DisksChanged({})","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.689312801Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Channel capacity increased","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.689327992Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:336","current":"255","previous":"245"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.689341602Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.689412095Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:111","resources":"StorageResources { disks: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: a462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal } }), Pool { name: ZpoolName { id: a462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal }, info: ZpoolInfo { name: \"oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b\", size: 20937965568, allocated: 497664, free: 20937467904, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: b462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal } }), Pool { name: ZpoolName { id: b462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal }, info: ZpoolInfo { name: \"oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b\", size: 20937965568, allocated: 496128, free: 20937469440, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" } })} }"} {"msg":"Starting Bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.689485087Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:200"} {"msg":"No ledger in /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.68957488Z","hostname":"buskin","pid":1671,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:187"} {"msg":"No ledger in /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.689609961Z","hostname":"buskin","pid":1671,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:187"} {"msg":"No ledger found. Loading Fsm::State as Uninitialized","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.689621551Z","hostname":"buskin","pid":1671,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:83"} {"msg":"No ledger in /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.689647402Z","hostname":"buskin","pid":1671,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:187"} {"msg":"No ledger in /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.689663933Z","hostname":"buskin","pid":1671,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:187"} {"msg":"No ledger found for network config","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.689670023Z","hostname":"buskin","pid":1671,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:146"} {"msg":"Starting Bootstore DDMD poller","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.689724235Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:205"} {"msg":"Starting ZoneBundler related tasks","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.689739715Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/long_running_tasks.rs:220"} {"msg":"Creating ServiceManager","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.704012874Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:697"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.704067946Z","hostname":"buskin","pid":1671,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:288","time_to_next_cleanup":"599.999936438s","next_cleanup":"Instant { tv_sec: 1266, tv_nsec: 303891939 }"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.704077426Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3345"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.704093317Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"No ledger in /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.704165989Z","hostname":"buskin","pid":1671,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:187"} {"msg":"No ledger in /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.70417871Z","hostname":"buskin","pid":1671,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:187"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.707445997Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:195"} {"msg":"Started listening","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.70754568Z","hostname":"buskin","pid":1671,"component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:44","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Injecting RSS configuration: RackInitializeRequest {\n trust_quorum_peers: None,\n bootstrap_discovery: OnlyOurs,\n ntp_servers: [\n \"0.pool.ntp.org\",\n ],\n dns_servers: [\n 1.1.1.1,\n 9.9.9.9,\n ],\n internal_services_ip_pool_ranges: [\n V4(\n Ipv4Range {\n first: 10.151.1.164,\n last: 10.151.1.173,\n },\n ),\n ],\n external_dns_ips: [\n 10.151.1.164,\n 10.151.1.165,\n ],\n external_dns_zone_name: \"oxide.test\",\n external_certificates: \"\",\n recovery_silo: RecoverySiloConfig {\n silo_name: Name(\n \"recovery\",\n ),\n user_name: UserId(\n \"recovery\",\n ),\n user_password_hash: NewPasswordHash(\n PasswordHashString {\n string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n encoding: B64,\n },\n ),\n },\n rack_network_config: RackNetworkConfigV1 {\n rack_subnet: Ipv6Network {\n addr: fd00:1122:3344:100::,\n prefix: 56,\n },\n infra_ip_first: 192.168.1.30,\n infra_ip_last: 192.168.1.30,\n ports: [\n PortConfigV1 {\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Network {\n addr: 0.0.0.0,\n prefix: 0,\n },\n ),\n nexthop: 10.151.1.100,\n },\n ],\n addresses: [\n V4(\n Ipv4Network {\n addr: 10.151.1.174,\n prefix: 24,\n },\n ),\n ],\n switch: Switch0,\n port: \"qsfp0\",\n uplink_port_speed: Speed40G,\n uplink_port_fec: None,\n bgp_peers: [],\n autoneg: false,\n },\n ],\n bgp: [],\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.707806229Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:800"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.7078266Z","hostname":"buskin","pid":1671,"component":"DnsResolver","component":"RSS","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.707895152Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"No ledger in /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.707950344Z","hostname":"buskin","pid":1671,"component":"RSS","file":"common/src/ledger.rs:187"} {"msg":"No ledger in /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.707970864Z","hostname":"buskin","pid":1671,"component":"RSS","file":"common/src/ledger.rs:187"} {"msg":"RSS configuration has not been fully applied yet","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.707978814Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:860"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.708010375Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"No ledger in /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.708024876Z","hostname":"buskin","pid":1671,"component":"RSS","file":"common/src/ledger.rs:187"} {"msg":"No ledger in /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.708034336Z","hostname":"buskin","pid":1671,"component":"RSS","file":"common/src/ledger.rs:187"} {"msg":"Creating new allocation plan","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.708048817Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:901"} {"msg":"Serializing plan","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.708051977Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:118"} {"msg":"Creating plan for the sled at fdb0:18c0:4d0d:9fb2::1","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.708054967Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:91"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.708078108Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.708099938Z","hostname":"buskin","pid":1671,"component":"RSS","file":"common/src/ledger.rs:198"} {"msg":"Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.708552533Z","hostname":"buskin","pid":1671,"component":"RSS","file":"common/src/ledger.rs:198"} {"msg":"Sled plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.70875938Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:138"} {"msg":"Writing Rack Network Configuration to bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.70877166Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:934"} {"msg":"Attempting to update network config with generation: 1, current_generation: 0","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.708819142Z","hostname":"buskin","pid":1671,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.708827252Z","hostname":"buskin","pid":1671,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:198"} {"msg":"Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.709007668Z","hostname":"buskin","pid":1671,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:198"} {"msg":"Broadcasting network config with generation 1","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.709174034Z","hostname":"buskin","pid":1671,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"Received initialization request from RSS","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.709227166Z","hostname":"buskin","pid":1671,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:217","target_sled":"[fdb0:18c0:4d0d:9fb2::1]:12346","request":"StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: ec2916b4-23be-4c03-a34b-23d4a0915a29, rack_id: ea034aee-f806-4033-865f-6a70bb9c7e3c, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net(Ipv6Network { addr: fd00:1122:3344:101::, prefix: 64 }) } } }"} {"msg":"Accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.709334209Z","hostname":"buskin","pid":1671,"remote_addr":"[fdb0:18c0:4d0d:9fb2::1]:58818","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:66"} {"msg":"Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: ec2916b4-23be-4c03-a34b-23d4a0915a29, rack_id: ea034aee-f806-4033-865f-6a70bb9c7e3c, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net(Ipv6Network { addr: fd00:1122:3344:101::, prefix: 64 }) } } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.709440413Z","hostname":"buskin","pid":1671,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:364"} {"msg":"KeyManager: using hardcoded secret retriever","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.709449633Z","hostname":"buskin","pid":1671,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:379"} {"msg":"setting up sled agent server","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.709464213Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/server.rs:47"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.709472263Z","hostname":"buskin","pid":1671,"component":"DnsResolver","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"Received KeyManagerReady","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.709477204Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Attempting to add queued disks","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.709481214Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:394","num_disks":"9"} {"msg":"Enabling switch zone (new)","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.711959885Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3573"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.718316844Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:327","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.718396527Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:329","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.718405877Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:333","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.718429628Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:335","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.718540741Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:37"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.718546802Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:347"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.749951053Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:56"} {"msg":"Automatically destroying dataset: oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.757888184Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:389"} {"msg":"SledAgent::new(..) starting","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.774302383Z","hostname":"buskin","pid":1671,"sled_id":"ec2916b4-23be-4c03-a34b-23d4a0915a29","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:342"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.774374715Z","hostname":"buskin","pid":1671,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:91","prefix":"Ipv6Prefix { addr: fd00:1122:3344:101::, len: 64 }"} {"msg":"Looking for unencrypted datasets in oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.858982655Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:474"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.882193507Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:327","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\",\n model: \"synthetic-model\",\n}"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.882241519Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:329","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.882254969Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:333","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.882337052Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:335","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.882450816Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:37"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.882459786Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:347"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.911343675Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:56"} {"msg":"Automatically destroying dataset: oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:27.919031017Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:389"} {"msg":"Looking for unencrypted datasets in oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:28.022990042Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:474"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:28.044732536Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:327","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:28.044773767Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:329","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:28.044788618Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:333","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:28.044805158Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:335","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:28.044959613Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:37"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:28.044969424Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:347"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:28.072663233Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:56"} {"msg":"Automatically destroying dataset: oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:28.080095848Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:389"} {"msg":"Looking for unencrypted datasets in oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:28.181465797Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:474"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:28.203418499Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:327","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\",\n model: \"synthetic-model\",\n}"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:28.203453829Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:329","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:28.20347806Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:333","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:28.203500301Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:335","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:28.203641506Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:37"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:28.203649736Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:347"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:28.231496981Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:56"} {"msg":"Automatically destroying dataset: oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:28.239240225Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:389"} {"msg":"Looking for unencrypted datasets in oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:28.34350115Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:474"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:28.366318439Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:327","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:28.366386882Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:329","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:28.366397272Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:333","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:28.366419493Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:335","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:28.366559337Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:37"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:28.366567827Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:347"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:28.393054008Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:56"} {"msg":"Automatically destroying dataset: oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:28.400298706Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:389"} {"msg":"Looking for unencrypted datasets in oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:28.501822341Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:474"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:28.524747034Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:327","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:28.524794015Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:329","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:28.524801945Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:333","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:28.524826036Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:335","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:28.524928769Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:37"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:28.52493529Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:347"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:28.55325733Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:56"} {"msg":"Automatically destroying dataset: oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:28.560810668Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:389"} {"msg":"Looking for unencrypted datasets in oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:28.667574345Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:474"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:28.690102775Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:327","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\",\n model: \"synthetic-model\",\n}"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:28.690145787Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:329","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:28.690169987Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:333","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:28.690197008Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:335","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:28.690323872Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:37"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:28.690331623Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:347"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:28.717900478Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:56"} {"msg":"Automatically destroying dataset: oxp_31bd71cd-4736-4a12-a387-9b74b050396f/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:28.724543717Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:389"} {"msg":"Looking for unencrypted datasets in oxp_31bd71cd-4736-4a12-a387-9b74b050396f","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:28.826572068Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:474"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:28.849774Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:327","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\",\n model: \"synthetic-model\",\n}"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:28.849830212Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:329","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:28.849839322Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:333","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:28.849875733Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:335","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:28.849995377Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:37"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:28.850002387Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:347"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:28.877824912Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:56"} {"msg":"Automatically destroying dataset: oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:28.885755432Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:389"} {"msg":"Looking for unencrypted datasets in oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:28.997323507Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:474"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:29.021505711Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:327","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\",\n model: \"synthetic-model\",\n}"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:29.021588964Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:329","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:29.021599134Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:333","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:29.021617745Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:335","disk_id":"DiskIdentity {\n vendor: \"synthetic-vendor\",\n serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\",\n model: \"synthetic-model\",\n}","epoch":"0"} {"msg":"Created keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:29.02176724Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:37"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:29.0217742Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:347"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60-synthetic-model-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:29.057680209Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/keyfile.rs:56"} {"msg":"Automatically destroying dataset: oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:29.065777065Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:389"} {"msg":"Looking for unencrypted datasets in oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:29.171376834Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/dataset.rs:474"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:29.186851052Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:29.186905184Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Requested swap device of size 64 GiB","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:29.186923895Z","hostname":"buskin","pid":1671,"sled_id":"ec2916b4-23be-4c03-a34b-23d4a0915a29","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:354"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:29.189683375Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:111","resources":"StorageResources { disks: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }), Pool { name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, info: ZpoolInfo { name: \"oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03\", size: 20937965568, allocated: 777216, free: 20937188352, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }), Pool { name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, info: ZpoolInfo { name: \"oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03\", size: 20937965568, allocated: 772608, free: 20937192960, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }), Pool { name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, info: ZpoolInfo { name: \"oxp_31bd71cd-4736-4a12-a387-9b74b050396f\", size: 20937965568, allocated: 774144, free: 20937191424, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } }), Pool { name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, info: ZpoolInfo { name: \"oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7\", size: 20937965568, allocated: 772608, free: 20937192960, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: a462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal } }), Pool { name: ZpoolName { id: a462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal }, info: ZpoolInfo { name: \"oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b\", size: 20937965568, allocated: 497664, free: 20937467904, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: b462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal } }), Pool { name: ZpoolName { id: b462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: Internal }, info: ZpoolInfo { name: \"oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b\", size: 20937965568, allocated: 496128, free: 20937469440, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } }), Pool { name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, info: ZpoolInfo { name: \"oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", size: 20937965568, allocated: 775680, free: 20937189888, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } }), Pool { name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, info: ZpoolInfo { name: \"oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60\", size: 20937965568, allocated: 1724928, free: 20936240640, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } }), Pool { name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, info: ZpoolInfo { name: \"oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b\", size: 20937965568, allocated: 769536, free: 20937196032, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }), Pool { name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, info: ZpoolInfo { name: \"oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", size: 20937965568, allocated: 823296, free: 20937142272, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" } }), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }: (Synthetic(SyntheticDisk { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }, zpool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }), Pool { name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, info: ZpoolInfo { name: \"oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", size: 20937965568, allocated: 775680, free: 20937189888, health: Online }, parent: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" } })} }"} {"msg":"attempting to create encrypted zvol: name=\"oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:29.209484616Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/swap_device.rs:153"} {"msg":"Configuring new Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:29.211821632Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:29.229354788Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"successfully created encrypted zvol: name=\"oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:29.232174761Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/swap_device.rs:249"} {"msg":"adding swap device: swapname=\"/dev/zvol/dsk/oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b/swap\"","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:29.232231803Z","hostname":"buskin","pid":1671,"file":"sled-agent/src/swap_device.rs:96"} {"msg":"Mounting backing filesystems","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:29.23700777Z","hostname":"buskin","pid":1671,"sled_id":"ec2916b4-23be-4c03-a34b-23d4a0915a29","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:369"} {"msg":"Processing fmd","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:29.23701821Z","hostname":"buskin","pid":1671,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:126"} {"msg":"Ensuring dataset oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b/backing/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:29.23702509Z","hostname":"buskin","pid":1671,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:136"} {"msg":"Stopping service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:29.297831298Z","hostname":"buskin","pid":1671,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:170"} {"msg":"Mounting oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b/backing/fmd on /var/fm/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:29.516111848Z","hostname":"buskin","pid":1671,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:178"} {"msg":"Ensuring directory /var/fm/fmd/rsrc","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:29.528408372Z","hostname":"buskin","pid":1671,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Ensuring directory /var/fm/fmd/ckpt","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:29.528497715Z","hostname":"buskin","pid":1671,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Ensuring directory /var/fm/fmd/xprt","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:29.528540146Z","hostname":"buskin","pid":1671,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Starting service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:29.528572527Z","hostname":"buskin","pid":1671,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:192"} {"msg":"Ensuring contract reaper thread","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:29.696468762Z","hostname":"buskin","pid":1671,"file":"illumos-utils/src/running_zone.rs:163"} {"msg":"using '[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]' as data links for xde driver","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:30.238160675Z","hostname":"buskin","pid":1671,"sled_id":"ec2916b4-23be-4c03-a34b-23d4a0915a29","component":"SledAgent","file":"illumos-utils/src/opte/illumos.rs:90"} {"msg":"Underlay Available","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:30.238590379Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:122","sled_id":"ec2916b4-23be-4c03-a34b-23d4a0915a29"} {"msg":"50% of 102952026112 physical ram = 51476013056 bytes)","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:30.23861786Z","hostname":"buskin","pid":1671,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:178"} {"msg":"Setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:30.23862527Z","hostname":"buskin","pid":1671,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:186"} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: ec2916b4-23be-4c03-a34b-23d4a0915a29, variant: M2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:30.251678749Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: ec2916b4-23be-4c03-a34b-23d4a0915a29, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:30.255947519Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: ec2916b4-23be-4c03-a34b-23d4a0915a29, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:30.256330652Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:30.256501667Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/ec2916b4-23be-4c03-a34b-23d4a0915a29/zpools/24b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:30.256720745Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/ec2916b4-23be-4c03-a34b-23d4a0915a29/zpools/b462a7f7-b628-40fe-80ff-4e4189e2d62b): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:30.256997134Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/ec2916b4-23be-4c03-a34b-23d4a0915a29/zpools/f4b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: ec2916b4-23be-4c03-a34b-23d4a0915a29, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:30.257166029Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", sled_id: ec2916b4-23be-4c03-a34b-23d4a0915a29, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:30.257284843Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", sled_id: ec2916b4-23be-4c03-a34b-23d4a0915a29, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:30.257443488Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: ec2916b4-23be-4c03-a34b-23d4a0915a29, variant: M2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:30.257584613Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", sled_id: ec2916b4-23be-4c03-a34b-23d4a0915a29, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:30.257725038Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", sled_id: ec2916b4-23be-4c03-a34b-23d4a0915a29, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:30.257862342Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: ec2916b4-23be-4c03-a34b-23d4a0915a29, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:30.258014167Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: ec2916b4-23be-4c03-a34b-23d4a0915a29, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:30.258151432Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:219","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:30.258279676Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/ec2916b4-23be-4c03-a34b-23d4a0915a29/zpools/14b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxp_31bd71cd-4736-4a12-a387-9b74b050396f on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:30.25841002Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/ec2916b4-23be-4c03-a34b-23d4a0915a29/zpools/31bd71cd-4736-4a12-a387-9b74b050396f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:30.258529974Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/ec2916b4-23be-4c03-a34b-23d4a0915a29/zpools/616b26df-e62a-4c68-b506-f4a923d8aaf7): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:30.258657398Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/ec2916b4-23be-4c03-a34b-23d4a0915a29/zpools/a462a7f7-b628-40fe-80ff-4e4189e2d62b): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:30.258764802Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/ec2916b4-23be-4c03-a34b-23d4a0915a29/zpools/cd70d7f6-2354-4bf2-8012-55bf9eaf7930): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:30.258888176Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/ec2916b4-23be-4c03-a34b-23d4a0915a29/zpools/ceb4461c-cf56-4719-ad3c-14430bfdfb60): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:30.25901487Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/ec2916b4-23be-4c03-a34b-23d4a0915a29/zpools/d462a7f7-b628-40fe-80ff-4e4189e2d62b): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"failed to notify nexus about a new pool oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:30.259123924Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:270","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/ec2916b4-23be-4c03-a34b-23d4a0915a29/zpools/e4b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:32.996859321Z","hostname":"buskin","pid":1671,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 69.407739ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:55:33.306710739Z","hostname":"buskin","pid":1671,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 120.031306ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:55:33.502837962Z","hostname":"buskin","pid":1671,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 178.645198ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:55:33.678529633Z","hostname":"buskin","pid":1671,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 467.164656ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:55:34.025909404Z","hostname":"buskin","pid":1671,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 743.006327ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:55:34.628590091Z","hostname":"buskin","pid":1671,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.171410088s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:55:35.447990036Z","hostname":"buskin","pid":1671,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.304091066s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:55:36.850102552Z","hostname":"buskin","pid":1671,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring tfportrear0_0/ll exists in zone","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:39.288324382Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1847"} {"msg":"sled agent started","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:40.457632651Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:978","underlay_address":"fd00:1122:3344:101::1"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:40.457737674Z","hostname":"buskin","pid":1671,"component":"DnsResolver","component":"ServiceManager","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:40.515939296Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:195"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:40.515983268Z","hostname":"buskin","pid":1671,"sled_id":"ec2916b4-23be-4c03-a34b-23d4a0915a29","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"ec2916b4-23be-4c03-a34b-23d4a0915a29"} {"msg":"Sled Agent loaded; recording configuration","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:40.515995868Z","hostname":"buskin","pid":1671,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:424"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:40.516015039Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:40.51605261Z","hostname":"buskin","pid":1671,"component":"SledAgentMain","file":"common/src/ledger.rs:198"} {"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:55:40.516197525Z","hostname":"buskin","pid":1671,"sled_id":"ec2916b4-23be-4c03-a34b-23d4a0915a29","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1199","retry_after":"280.112816ms","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:40.51635371Z","hostname":"buskin","pid":1671,"component":"SledAgentMain","file":"common/src/ledger.rs:198"} {"msg":"Sled Agent Started","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:40.516530046Z","hostname":"buskin","pid":1671,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:137"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:40.516558167Z","hostname":"buskin","pid":1671,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:241","underlay_network_info":"Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV1 { rack_subnet: Ipv6Network { addr: fd00:1122:3344:100::, prefix: 56 }, infra_ip_first: 192.168.1.30, infra_ip_last: 192.168.1.30, ports: [PortConfigV1 { routes: [RouteConfig { destination: V4(Ipv4Network { addr: 0.0.0.0, prefix: 0 }), nexthop: 10.151.1.100 }], addresses: [V4(Ipv4Network { addr: 10.151.1.174, prefix: 24 })], switch: Switch0, port: \"qsfp0\", uplink_port_speed: Speed40G, uplink_port_fec: None, bgp_peers: [], autoneg: false }], bgp: [] })))"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:40.516569617Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3345"} {"msg":"Connection closed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:40.516575677Z","hostname":"buskin","pid":1671,"remote_addr":"[fdb0:18c0:4d0d:9fb2::1]:58818","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:73"} {"msg":"Peer agent initialized","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:40.519553005Z","hostname":"buskin","pid":1671,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:108","peer":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Initialized sled agent","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:40.519622197Z","hostname":"buskin","pid":1671,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:232","target_sled":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:40.519762612Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"No ledger in /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/rss-service-plan-v2.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:40.519899016Z","hostname":"buskin","pid":1671,"component":"RSS","file":"common/src/ledger.rs:187"} {"msg":"No ledger in /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/rss-service-plan-v2.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:40.519911957Z","hostname":"buskin","pid":1671,"component":"RSS","file":"common/src/ledger.rs:187"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:40.519949598Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"failed to notify nexus about sled agent","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:40.520355751Z","hostname":"buskin","pid":1671,"sled_id":"ec2916b4-23be-4c03-a34b-23d4a0915a29","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:735","error":"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/ec2916b4-23be-4c03-a34b-23d4a0915a29): error trying to connect: dns error: proto error: io error: No route to host (os error 148)"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:40.55257695Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:64543"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:40.552845829Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:40.552932572Z","hostname":"buskin","pid":1671,"uri":"/zpools","method":"GET","req_id":"3be9d4e9-b580-4cd9-83b8-ccfd32e28ca5","remote_addr":"[fd00:1122:3344:101::1]:64543","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":207,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:40.58331351Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:46421"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:40.583428723Z","hostname":"buskin","pid":1671,"uri":"/sled-role","method":"GET","req_id":"88769b0f-3d15-4b4e-a737-ce633d56da5e","remote_addr":"[fd00:1122:3344:101::1]:46421","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":65,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:40.585665807Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.rss-service-plan-v2.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:40.585677667Z","hostname":"buskin","pid":1671,"component":"RSS","file":"common/src/ledger.rs:198"} {"msg":"Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.rss-service-plan-v2.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:40.585965706Z","hostname":"buskin","pid":1671,"component":"RSS","file":"common/src/ledger.rs:198"} {"msg":"Service plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:40.586324078Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/plan/service.rs:748"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:40.602342695Z","hostname":"buskin","pid":1671,"sled_id":"ec2916b4-23be-4c03-a34b-23d4a0915a29","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"ec2916b4-23be-4c03-a34b-23d4a0915a29"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 719635e7-d366-4895-9272-9ade9dc77ba8, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2ae10e87-0030-442b-9981-0b4c70a9c369, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:40.617386199Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:40.617573325Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:59412"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:40.617842734Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:40.617857944Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 719635e7-d366-4895-9272-9ade9dc77ba8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:40.671230427Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 719635e7-d366-4895-9272-9ade9dc77ba8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:40.671271599Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:40.691218264Z","hostname":"buskin","pid":1671,"sled_id":"ec2916b4-23be-4c03-a34b-23d4a0915a29","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"ec2916b4-23be-4c03-a34b-23d4a0915a29"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ae10e87-0030-442b-9981-0b4c70a9c369, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:40.721715956Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2ae10e87-0030-442b-9981-0b4c70a9c369, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:40.721753407Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:40.770140726Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"No ledger in /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:40.770180878Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:187"} {"msg":"No ledger in /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:40.770188838Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:187"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:40.77023951Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:55:40.8000865Z","hostname":"buskin","pid":1671,"sled_id":"ec2916b4-23be-4c03-a34b-23d4a0915a29","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1199","retry_after":"684.16509ms","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:40.81378539Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:40.813845862Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:40.813852222Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_719635e7-d366-4895-9272-9ade9dc77ba8","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:40.867717122Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_6322d35a-04fa-4d40-ba3c-fb0208e2f2f4","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:40.871837417Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_2ae10e87-0030-442b-9981-0b4c70a9c369","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:40.875921321Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_internal_dns_719635e7-d366-4895-9272-9ade9dc77ba8","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:40.888098411Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_6322d35a-04fa-4d40-ba3c-fb0208e2f2f4","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:40.89293031Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:40.897407247Z","hostname":"buskin","pid":1671,"sled_id":"ec2916b4-23be-4c03-a34b-23d4a0915a29","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"ec2916b4-23be-4c03-a34b-23d4a0915a29"} {"msg":"Installing Omicron zone: oxz_internal_dns_2ae10e87-0030-442b-9981-0b4c70a9c369","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:40.897532541Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:41.013643185Z","hostname":"buskin","pid":1671,"sled_id":"ec2916b4-23be-4c03-a34b-23d4a0915a29","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"ec2916b4-23be-4c03-a34b-23d4a0915a29"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:41.145866248Z","hostname":"buskin","pid":1671,"sled_id":"ec2916b4-23be-4c03-a34b-23d4a0915a29","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"ec2916b4-23be-4c03-a34b-23d4a0915a29"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:41.391787846Z","hostname":"buskin","pid":1671,"sled_id":"ec2916b4-23be-4c03-a34b-23d4a0915a29","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"ec2916b4-23be-4c03-a34b-23d4a0915a29"} {"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:55:41.486345262Z","hostname":"buskin","pid":1671,"sled_id":"ec2916b4-23be-4c03-a34b-23d4a0915a29","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1199","retry_after":"542.876488ms","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:41.719363326Z","hostname":"buskin","pid":1671,"sled_id":"ec2916b4-23be-4c03-a34b-23d4a0915a29","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"ec2916b4-23be-4c03-a34b-23d4a0915a29"} {"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:55:42.031274542Z","hostname":"buskin","pid":1671,"sled_id":"ec2916b4-23be-4c03-a34b-23d4a0915a29","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1199","retry_after":"1.926809926s","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:42.145844655Z","hostname":"buskin","pid":1671,"sled_id":"ec2916b4-23be-4c03-a34b-23d4a0915a29","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"ec2916b4-23be-4c03-a34b-23d4a0915a29"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:42.669426604Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_719635e7-d366-4895-9272-9ade9dc77ba8","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:42.706527743Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_2ae10e87-0030-442b-9981-0b4c70a9c369","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:42.716066646Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_6322d35a-04fa-4d40-ba3c-fb0208e2f2f4","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:42.731549315Z","hostname":"buskin","pid":1671,"sled_id":"ec2916b4-23be-4c03-a34b-23d4a0915a29","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"ec2916b4-23be-4c03-a34b-23d4a0915a29"} {"msg":"wait for service Some(\"oxz_internal_dns_719635e7-d366-4895-9272-9ade9dc77ba8\") failed: Property not found. retry in 47.45635ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:55:42.801808472Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_719635e7-d366-4895-9272-9ade9dc77ba8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_2ae10e87-0030-442b-9981-0b4c70a9c369\") failed: Property not found. retry in 64.495831ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:55:42.836925956Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_2ae10e87-0030-442b-9981-0b4c70a9c369","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_6322d35a-04fa-4d40-ba3c-fb0208e2f2f4\") failed: Property not found. retry in 38.286759ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:55:42.844524215Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_6322d35a-04fa-4d40-ba3c-fb0208e2f2f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_719635e7-d366-4895-9272-9ade9dc77ba8\") failed: Property not found. retry in 92.080994ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:55:42.857548393Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_719635e7-d366-4895-9272-9ade9dc77ba8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_6322d35a-04fa-4d40-ba3c-fb0208e2f2f4\") failed: Property not found. retry in 137.337339ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:55:42.901052992Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_6322d35a-04fa-4d40-ba3c-fb0208e2f2f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_2ae10e87-0030-442b-9981-0b4c70a9c369\") failed: Property not found. retry in 122.172006ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:55:43.015695367Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_2ae10e87-0030-442b-9981-0b4c70a9c369","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_719635e7-d366-4895-9272-9ade9dc77ba8\") failed: Property not found. retry in 249.136127ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:55:43.126158066Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_719635e7-d366-4895-9272-9ade9dc77ba8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_6322d35a-04fa-4d40-ba3c-fb0208e2f2f4\") failed: Property not found. retry in 173.31993ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:55:43.200121055Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_6322d35a-04fa-4d40-ba3c-fb0208e2f2f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_2ae10e87-0030-442b-9981-0b4c70a9c369\") failed: Property not found. retry in 195.789433ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:55:43.276556156Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_2ae10e87-0030-442b-9981-0b4c70a9c369","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:43.309708265Z","hostname":"buskin","pid":1671,"sled_id":"ec2916b4-23be-4c03-a34b-23d4a0915a29","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"ec2916b4-23be-4c03-a34b-23d4a0915a29"} {"msg":"wait for service Some(\"oxz_internal_dns_6322d35a-04fa-4d40-ba3c-fb0208e2f2f4\") failed: Property not found. retry in 575.315485ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:55:43.417173175Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_6322d35a-04fa-4d40-ba3c-fb0208e2f2f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_719635e7-d366-4895-9272-9ade9dc77ba8\") failed: Property not found. retry in 379.759994ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:55:43.455803384Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_719635e7-d366-4895-9272-9ade9dc77ba8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_2ae10e87-0030-442b-9981-0b4c70a9c369\") failed: Property not found. retry in 379.429876ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:55:43.522071021Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_2ae10e87-0030-442b-9981-0b4c70a9c369","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:43.822925113Z","hostname":"buskin","pid":1671,"sled_id":"ec2916b4-23be-4c03-a34b-23d4a0915a29","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"ec2916b4-23be-4c03-a34b-23d4a0915a29"} {"msg":"wait for service Some(\"oxz_internal_dns_719635e7-d366-4895-9272-9ade9dc77ba8\") failed: Property not found. retry in 1.16858165s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:55:43.845572357Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_719635e7-d366-4895-9272-9ade9dc77ba8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_2ae10e87-0030-442b-9981-0b4c70a9c369\") failed: Property not found. retry in 945.718548ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:55:43.909572319Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_2ae10e87-0030-442b-9981-0b4c70a9c369","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:55:43.960660488Z","hostname":"buskin","pid":1671,"sled_id":"ec2916b4-23be-4c03-a34b-23d4a0915a29","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1199","retry_after":"2.253185968s","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"wait for service Some(\"oxz_internal_dns_6322d35a-04fa-4d40-ba3c-fb0208e2f2f4\") failed: Property not found. retry in 463.599677ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:55:44.001589142Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_6322d35a-04fa-4d40-ba3c-fb0208e2f2f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring bootstrap address fdb0:18c0:4d0d:9fb2::2 exists in switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:44.420264214Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1864"} {"msg":"Adding bootstrap address","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:44.420287745Z","hostname":"buskin","pid":1671,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:626"} {"msg":"wait for service Some(\"oxz_internal_dns_6322d35a-04fa-4d40-ba3c-fb0208e2f2f4\") failed: Property not found. retry in 963.61236ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:55:44.47553373Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_6322d35a-04fa-4d40-ba3c-fb0208e2f2f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:44.498628179Z","hostname":"buskin","pid":1671,"sled_id":"ec2916b4-23be-4c03-a34b-23d4a0915a29","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"ec2916b4-23be-4c03-a34b-23d4a0915a29"} {"msg":"wait for service Some(\"oxz_internal_dns_2ae10e87-0030-442b-9981-0b4c70a9c369\") failed: Property not found. retry in 1.392916252s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:55:44.866291636Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_2ae10e87-0030-442b-9981-0b4c70a9c369","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_719635e7-d366-4895-9272-9ade9dc77ba8\") failed: Property not found. retry in 1.262193205s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:55:45.025399841Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_719635e7-d366-4895-9272-9ade9dc77ba8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_6322d35a-04fa-4d40-ba3c-fb0208e2f2f4\") failed: Property not found. retry in 751.13672ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:55:45.449462491Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_6322d35a-04fa-4d40-ba3c-fb0208e2f2f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:45.817984526Z","hostname":"buskin","pid":1671,"sled_id":"ec2916b4-23be-4c03-a34b-23d4a0915a29","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"ec2916b4-23be-4c03-a34b-23d4a0915a29"} {"msg":"wait for service Some(\"oxz_internal_dns_6322d35a-04fa-4d40-ba3c-fb0208e2f2f4\") failed: Property not found. retry in 610.324881ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:55:46.209814927Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_6322d35a-04fa-4d40-ba3c-fb0208e2f2f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:55:46.21507124Z","hostname":"buskin","pid":1671,"sled_id":"ec2916b4-23be-4c03-a34b-23d4a0915a29","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1199","retry_after":"9.657006932s","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""} {"msg":"wait for service Some(\"oxz_internal_dns_2ae10e87-0030-442b-9981-0b4c70a9c369\") failed: Property not found. retry in 1.33876657s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:55:46.267846313Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_2ae10e87-0030-442b-9981-0b4c70a9c369","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_719635e7-d366-4895-9272-9ade9dc77ba8\") failed: Property not found. retry in 853.67524ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:55:46.29545673Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_719635e7-d366-4895-9272-9ade9dc77ba8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:46.643162562Z","hostname":"buskin","pid":1671,"sled_id":"ec2916b4-23be-4c03-a34b-23d4a0915a29","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"ec2916b4-23be-4c03-a34b-23d4a0915a29"} {"msg":"wait for service Some(\"oxz_internal_dns_6322d35a-04fa-4d40-ba3c-fb0208e2f2f4\") failed: Property not found. retry in 639.758197ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:55:46.829713509Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_6322d35a-04fa-4d40-ba3c-fb0208e2f2f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_719635e7-d366-4895-9272-9ade9dc77ba8\") failed: Property not found. retry in 841.52114ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:55:47.159208273Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_719635e7-d366-4895-9272-9ade9dc77ba8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_6322d35a-04fa-4d40-ba3c-fb0208e2f2f4\") failed: Property not found. retry in 1.474703243s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:55:47.478453389Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_6322d35a-04fa-4d40-ba3c-fb0208e2f2f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_2ae10e87-0030-442b-9981-0b4c70a9c369\") failed: Property not found. retry in 1.40871103s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:55:47.615230962Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_2ae10e87-0030-442b-9981-0b4c70a9c369","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:47.680297619Z","hostname":"buskin","pid":1671,"sled_id":"ec2916b4-23be-4c03-a34b-23d4a0915a29","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"ec2916b4-23be-4c03-a34b-23d4a0915a29"} {"msg":"Ensuring address fd00:1122:3344:2::1 exists","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:48.047737408Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1900"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:2::1, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:48.04778745Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_719635e7-d366-4895-9272-9ade9dc77ba8","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"Ensuring address fd00:1122:3344:2::1 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:48.612836191Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1908"} {"msg":"Zone not using gateway (even though underlay is up)","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:48.612890263Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1932"} {"msg":"Setting up internal-dns service","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:48.781669967Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2126"} {"msg":"Forwarding bootstrap traffic via oxBootstrap0 to fe80::8:20ff:fe82:a505","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:48.952264111Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1871"} {"msg":"Zone not using gateway (even though underlay is up)","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:48.958134273Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1932"} {"msg":"Setting up dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:49.077799003Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2404"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:49.313559367Z","hostname":"buskin","pid":1671,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:91","prefix":"Ipv6Prefix { addr: fd00:1122:3344:2::, len: 64 }"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:49.313853337Z","hostname":"buskin","pid":1671,"sled_id":"ec2916b4-23be-4c03-a34b-23d4a0915a29","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"ec2916b4-23be-4c03-a34b-23d4a0915a29"} {"msg":"Setting up MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:49.328783108Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2283"} {"msg":"Ensuring address fd00:1122:3344:1::1 exists","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:49.362570648Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1900"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:1::1, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:49.362629639Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_6322d35a-04fa-4d40-ba3c-fb0208e2f2f4","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"Setting up wicketd service","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:49.624671867Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2318"} {"msg":"Setting up mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:49.814696699Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2585"} {"msg":"Ensuring address fd00:1122:3344:1::1 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:49.909191433Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1908"} {"msg":"Zone not using gateway (even though underlay is up)","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:49.909237425Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1932"} {"msg":"Setting up mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:49.958894996Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2590"} {"msg":"Setting up internal-dns service","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:50.087298673Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2126"} {"msg":"Setting up tfport service","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:50.140901454Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2524"} {"msg":"Setting up Simulated SP service","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:50.306290567Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2312"} {"msg":"Re-enabling running switch zone (new address)","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:50.315925203Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3597","new":"[fd00:1122:3344:101::2, ::1]","old":"[::1]"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:50.315963434Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3613"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:50.315973755Z","hostname":"buskin","pid":1671,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"Ensuring address fd00:1122:3344:3::1 exists","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:50.643332358Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1900"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:3::1, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:50.643376189Z","hostname":"buskin","pid":1671,"zone":"oxz_internal_dns_2ae10e87-0030-442b-9981-0b4c70a9c369","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:50.860082888Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3621"} {"msg":"configuring dendrite zone","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:50.866085955Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3677"} {"msg":"configuring wicketd","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:51.047563506Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3717","rack_subnet":"fd00:1122:3344::"} {"msg":"Determining physical location of our switch zone at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:51.100704622Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:360"} {"msg":"Failed to get switch ID from MGS (retrying in 50.314043ms)","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:55:51.132812976Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:379","error":"Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"60b799e1-0f26-4f54-a916-6f44c7576517\", \"content-length\": \"152\", \"date\": \"Thu, 15 Feb 2024 05:55:50 GMT\"}; value: Error { error_code: Some(\"DiscoveryNotYetComplete\"), message: \"discovery process not yet complete\", request_id: \"60b799e1-0f26-4f54-a916-6f44c7576517\" }"} {"msg":"Failed to get switch ID from MGS (retrying in 58.292822ms)","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:55:51.184526255Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:379","error":"Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"50981c83-fd0f-493b-be29-e8dd2e9990c1\", \"content-length\": \"152\", \"date\": \"Thu, 15 Feb 2024 05:55:50 GMT\"}; value: Error { error_code: Some(\"DiscoveryNotYetComplete\"), message: \"discovery process not yet complete\", request_id: \"50981c83-fd0f-493b-be29-e8dd2e9990c1\" }"} {"msg":"Ensuring address fd00:1122:3344:3::1 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:51.192168786Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1908"} {"msg":"Zone not using gateway (even though underlay is up)","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:51.192182087Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1932"} {"msg":"Failed to get switch ID from MGS (retrying in 118.096568ms)","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:55:51.243801662Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:379","error":"Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5a29eea4-ffcf-4947-adb0-c8250b9507ee\", \"content-length\": \"152\", \"date\": \"Thu, 15 Feb 2024 05:55:50 GMT\"}; value: Error { error_code: Some(\"DiscoveryNotYetComplete\"), message: \"discovery process not yet complete\", request_id: \"5a29eea4-ffcf-4947-adb0-c8250b9507ee\" }"} {"msg":"Failed to get switch ID from MGS (retrying in 387.648511ms)","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:55:51.363138122Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:379","error":"Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1df27e28-76df-4a80-80da-14aa05965ccc\", \"content-length\": \"152\", \"date\": \"Thu, 15 Feb 2024 05:55:50 GMT\"}; value: Error { error_code: Some(\"DiscoveryNotYetComplete\"), message: \"discovery process not yet complete\", request_id: \"1df27e28-76df-4a80-80da-14aa05965ccc\" }"} {"msg":"Setting up internal-dns service","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:51.367125653Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2126"} {"msg":"Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:51.751284342Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:408"} {"msg":"Building Port Configuration","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:51.78350306Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:567"} {"msg":"Checking dendrite uptime","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:51.783592383Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:609"} {"msg":"Dendrite online","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:51.784339478Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:612","uptime":"2"} {"msg":"Configuring default uplink on switch","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:51.784380059Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:430","config":"PortSettings {\n links: {\n \"0\": LinkSettings {\n addrs: [\n 10.151.1.174,\n ],\n params: LinkCreate {\n autoneg: false,\n fec: None,\n kr: false,\n lane: Some(\n LinkId(\n 0,\n ),\n ),\n speed: Speed40G,\n },\n },\n },\n v4_routes: {},\n v6_routes: {},\n}"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:51.846234691Z","hostname":"buskin","pid":1671,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:91","prefix":"Ipv6Prefix { addr: fd00:1122:3344:1::, len: 64 }"} {"msg":"Received DisksChanged({})","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:51.865250596Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:51.896513272Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_internal_dns_719635e7-d366-4895-9272-9ade9dc77ba8"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:51.896575454Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_internal_dns_6322d35a-04fa-4d40-ba3c-fb0208e2f2f4"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:51.896583645Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_internal_dns_2ae10e87-0030-442b-9981-0b4c70a9c369"} {"msg":"Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:51.896609166Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:198"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:51.8967328Z","hostname":"buskin","pid":1671,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:91","prefix":"Ipv6Prefix { addr: fd00:1122:3344:3::, len: 64 }"} {"msg":"Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:51.896978188Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:198"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:51.897297318Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"6075b4c6-00b8-4da3-8f76-130f803ed3fc","remote_addr":"[fd00:1122:3344:101::1]:59412","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":11279672,"response_code":"204"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:51.899726368Z","hostname":"buskin","pid":1671,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:417"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:51.947104044Z","hostname":"buskin","pid":1671,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:417"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:51.994933945Z","hostname":"buskin","pid":1671,"dns_config_addr":"[fd00:1122:3344:3::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:417"} {"msg":"Configured all DNS servers","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:52.52114703Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:453"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:52.521165561Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:254"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:52.551066643Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:271","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:52.552299884Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:296"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:52.552323165Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:188","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 719635e7-d366-4895-9272-9ade9dc77ba8, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2ae10e87-0030-442b-9981-0b4c70a9c369, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a62bfe46-059b-4ea1-b2a5-7f98d258138d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 44034536-a7b3-4ef5-9ea6-8cd61921d1ee, kind: Service { id: a62bfe46-059b-4ea1-b2a5-7f98d258138d }, name: Name(\"ntp-a62bfe46-059b-4ea1-b2a5-7f98d258138d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 159])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:52.580327954Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:52.580535301Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:38152"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:52.58079505Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:52.580814781Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 719635e7-d366-4895-9272-9ade9dc77ba8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:52.602637277Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 719635e7-d366-4895-9272-9ade9dc77ba8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:52.602716Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ae10e87-0030-442b-9981-0b4c70a9c369, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:52.623009027Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2ae10e87-0030-442b-9981-0b4c70a9c369, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:52.625124866Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:52.642175416Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:52.642244698Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:52.642434205Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:52.642539668Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:52.653685514Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:254"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:52.684777205Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:271","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:52.686202272Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:296"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:52.686217063Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:188","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:52.730532578Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte0\", _ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 138, 159]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 }, vnic: \"vopte0\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:52.73058814Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1268","zone_type":"ntp"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.1.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:52.731120968Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:98"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.1.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:52.731132818Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:105"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.1.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:52.731405917Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:133"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:52.732777222Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Configuring new Omicron zone: oxz_ntp_a62bfe46-059b-4ea1-b2a5-7f98d258138d","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:52.757334719Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_ntp_a62bfe46-059b-4ea1-b2a5-7f98d258138d","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:52.775406562Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:53.791635023Z","hostname":"buskin","pid":1671,"zone":"oxz_ntp_a62bfe46-059b-4ea1-b2a5-7f98d258138d","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_ntp_a62bfe46-059b-4ea1-b2a5-7f98d258138d\") failed: Property not found. retry in 40.152515ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:55:53.922496442Z","hostname":"buskin","pid":1671,"zone":"oxz_ntp_a62bfe46-059b-4ea1-b2a5-7f98d258138d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_a62bfe46-059b-4ea1-b2a5-7f98d258138d\") failed: Property not found. retry in 56.56981ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:55:53.969219607Z","hostname":"buskin","pid":1671,"zone":"oxz_ntp_a62bfe46-059b-4ea1-b2a5-7f98d258138d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_a62bfe46-059b-4ea1-b2a5-7f98d258138d\") failed: Property not found. retry in 155.176658ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:55:54.036564689Z","hostname":"buskin","pid":1671,"zone":"oxz_ntp_a62bfe46-059b-4ea1-b2a5-7f98d258138d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_a62bfe46-059b-4ea1-b2a5-7f98d258138d\") failed: Property not found. retry in 447.117243ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:55:54.202908173Z","hostname":"buskin","pid":1671,"zone":"oxz_ntp_a62bfe46-059b-4ea1-b2a5-7f98d258138d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_a62bfe46-059b-4ea1-b2a5-7f98d258138d\") failed: Property not found. retry in 1.073158047s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:55:54.660416721Z","hostname":"buskin","pid":1671,"zone":"oxz_ntp_a62bfe46-059b-4ea1-b2a5-7f98d258138d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:54.902310757Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:45025"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:54.902512753Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"563a5428-daa1-4c1f-b42c-681a3ca761a1","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:45025","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":92,"response_code":"200"} {"msg":"wait for service Some(\"oxz_ntp_a62bfe46-059b-4ea1-b2a5-7f98d258138d\") failed: Property not found. retry in 556.394199ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:55:55.742739322Z","hostname":"buskin","pid":1671,"zone":"oxz_ntp_a62bfe46-059b-4ea1-b2a5-7f98d258138d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_a62bfe46-059b-4ea1-b2a5-7f98d258138d\") failed: Property not found. retry in 1.270647386s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:55:56.309142537Z","hostname":"buskin","pid":1671,"zone":"oxz_ntp_a62bfe46-059b-4ea1-b2a5-7f98d258138d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_a62bfe46-059b-4ea1-b2a5-7f98d258138d\") failed: Property not found. retry in 957.724997ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:55:57.589182014Z","hostname":"buskin","pid":1671,"zone":"oxz_ntp_a62bfe46-059b-4ea1-b2a5-7f98d258138d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_a62bfe46-059b-4ea1-b2a5-7f98d258138d\") failed: Property not found. retry in 825.657681ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:55:58.556957424Z","hostname":"buskin","pid":1671,"zone":"oxz_ntp_a62bfe46-059b-4ea1-b2a5-7f98d258138d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring address fd00:1122:3344:101::1b exists","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:59.424072397Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1900"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::1b, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:59.424138899Z","hostname":"buskin","pid":1671,"zone":"oxz_ntp_a62bfe46-059b-4ea1-b2a5-7f98d258138d","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"Ensuring address fd00:1122:3344:101::1b exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:59.973710861Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1908"} {"msg":"Zone using sled underlay as gateway","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:55:59.973768253Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1927"} {"msg":"Set up NTP service boundary=true, Servers=[\"0.pool.ntp.org\"]","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:00.369565474Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2209"} {"msg":"Ensuring address for OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:00.400956945Z","hostname":"buskin","pid":1671,"zone":"oxz_ntp_a62bfe46-059b-4ea1-b2a5-7f98d258138d","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:653"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:06.798383976Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_ntp_a62bfe46-059b-4ea1-b2a5-7f98d258138d"} {"msg":"Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:06.7984914Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:198"} {"msg":"Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:06.79880341Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:198"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:06.799045888Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"89ac1c17-58f4-491d-9d10-116ae7e8589f","remote_addr":"[fd00:1122:3344:101::1]:38152","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":14218459,"response_code":"204"} {"msg":"Waiting for rack time synchronization","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:06.801166787Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:494"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:06.829740006Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:06.829900551Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:51176"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_a62bfe46-059b-4ea1-b2a5-7f98d258138d': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:06.839969252Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3331"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:06.840148808Z","hostname":"buskin","pid":1671,"uri":"/timesync","method":"GET","req_id":"c0f64fbd-cb1d-45b9-be5b-051fb3308643","remote_addr":"[fd00:1122:3344:101::1]:51176","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":10116,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:06.842470914Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:06.991004712Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:06.991135756Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:45227"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:06.999235033Z","hostname":"buskin","pid":1671,"uri":"/timesync","method":"GET","req_id":"a06bb64c-e49f-4691-8429-f9216c84c4c1","remote_addr":"[fd00:1122:3344:101::1]:45227","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":7983,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1707976565.868991, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:07.002521601Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:07.002540731Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:07.160098417Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:07.160228721Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:58901"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:07.168586445Z","hostname":"buskin","pid":1671,"uri":"/timesync","method":"GET","req_id":"752600d4-9503-46d7-9927-051a4ddc1e41","remote_addr":"[fd00:1122:3344:101::1]:58901","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":8215,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1707976565.868991, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:07.171083278Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:07.171096908Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:07.386810544Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:07.386909077Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:50286"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:07.394570299Z","hostname":"buskin","pid":1671,"uri":"/timesync","method":"GET","req_id":"0eb9859f-f378-4024-951f-23a124e67e57","remote_addr":"[fd00:1122:3344:101::1]:50286","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":7565,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1707976565.868991, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:07.396940466Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:07.396954557Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:07.663867894Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:07.663965618Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:59330"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:07.670960227Z","hostname":"buskin","pid":1671,"uri":"/timesync","method":"GET","req_id":"a68c05ad-8117-4bea-8247-92e112b474a6","remote_addr":"[fd00:1122:3344:101::1]:59330","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":6899,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1707976565.868991, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:07.674158423Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:07.674166863Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:07.973469514Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:07.973577788Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:63392"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:07.980537166Z","hostname":"buskin","pid":1671,"uri":"/timesync","method":"GET","req_id":"9bac040c-0576-49cc-a668-a03e6cde7b14","remote_addr":"[fd00:1122:3344:101::1]:63392","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":6870,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1707976565.868991, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:07.983539065Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:07.983572556Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:08.213826609Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:08.213919212Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:61057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:08.222114642Z","hostname":"buskin","pid":1671,"uri":"/timesync","method":"GET","req_id":"c82bc08b-b3bf-43a0-a9d9-00ca0a4c1c84","remote_addr":"[fd00:1122:3344:101::1]:61057","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":8107,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1707976565.868991, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:08.223846859Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:08.223852709Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:08.51888153Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:08.518978173Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:47478"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:08.52741319Z","hostname":"buskin","pid":1671,"uri":"/timesync","method":"GET","req_id":"8c00806b-7869-4f12-8e65-1fb737537d36","remote_addr":"[fd00:1122:3344:101::1]:47478","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":8276,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1707976565.868991, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:08.529011463Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:08.529017623Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:08.877689906Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:08.877775269Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:59357"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:08.885913926Z","hostname":"buskin","pid":1671,"uri":"/timesync","method":"GET","req_id":"ce518efb-2c72-4c51-9b84-767ef8551127","remote_addr":"[fd00:1122:3344:101::1]:59357","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":8006,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1707976565.868991, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:08.887086545Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:08.887094535Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:09.404852232Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:09.404944395Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:33564"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:09.413502236Z","hostname":"buskin","pid":1671,"uri":"/timesync","method":"GET","req_id":"bc6ae51e-0958-4fb1-993a-88264bfff274","remote_addr":"[fd00:1122:3344:101::1]:33564","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":8435,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1707976565.868991, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:09.415109149Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:09.415116969Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:09.929041641Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:09.929123354Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:47007"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:09.937563541Z","hostname":"buskin","pid":1671,"uri":"/timesync","method":"GET","req_id":"e5c325a8-7a3c-4003-b956-364cad7d8029","remote_addr":"[fd00:1122:3344:101::1]:47007","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":8328,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1707976565.868991, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:09.941095927Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:09.941104567Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:10.717667785Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:10.717774258Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:41408"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:10.725958857Z","hostname":"buskin","pid":1671,"uri":"/timesync","method":"GET","req_id":"17dbdc11-1e10-4242-8e22-6b62882e0e0e","remote_addr":"[fd00:1122:3344:101::1]:41408","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":8097,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1707976565.868991, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:10.727225539Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:10.727233829Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:11.500079735Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:11.500274382Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:43875"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:11.508607185Z","hostname":"buskin","pid":1671,"uri":"/timesync","method":"GET","req_id":"2ed56ba1-cfda-412d-9c51-d8c3eb2ef7c3","remote_addr":"[fd00:1122:3344:101::1]:43875","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":8277,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1707976565.868991, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:11.511677426Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:11.511685797Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:524","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:12.348773056Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:474"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:12.348862299Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:39003"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:12.351667441Z","hostname":"buskin","pid":1671,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:776"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:12.369060712Z","hostname":"buskin","pid":1671,"uri":"/timesync","method":"GET","req_id":"f345f93e-7056-44f0-8793-4a1c2aa478f0","remote_addr":"[fd00:1122:3344:101::1]:39003","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":20072,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: true, ref_id: 750659707, ip_addr: 44.190.40.123, stratum: 3, ref_time: 1707976571.113948, correction: -0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:12.37080747Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:502"} {"msg":"Finished setting up Internal DNS and NTP","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:12.37081595Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1039"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 719635e7-d366-4895-9272-9ade9dc77ba8, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2ae10e87-0030-442b-9981-0b4c70a9c369, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a62bfe46-059b-4ea1-b2a5-7f98d258138d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 44034536-a7b3-4ef5-9ea6-8cd61921d1ee, kind: Service { id: a62bfe46-059b-4ea1-b2a5-7f98d258138d }, name: Name(\"ntp-a62bfe46-059b-4ea1-b2a5-7f98d258138d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 159])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bf26395e-9671-4bee-b96d-1c6f76e85a02, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 21544c6f-59af-4a59-a720-d40b64e76f5d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: ddb8e0b6-c193-419e-b893-def90acf2c6e, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:12.39972738Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:12.399947577Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:61010"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:12.400152764Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:12.400171314Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 719635e7-d366-4895-9272-9ade9dc77ba8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:12.420415229Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 719635e7-d366-4895-9272-9ade9dc77ba8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:12.420472991Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ae10e87-0030-442b-9981-0b4c70a9c369, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:12.439524627Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2ae10e87-0030-442b-9981-0b4c70a9c369, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:12.439594289Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:12.457661473Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:12.457718555Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bf26395e-9671-4bee-b96d-1c6f76e85a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:12.548684223Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bf26395e-9671-4bee-b96d-1c6f76e85a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:12.548748615Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:12.634422149Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:12.636774626Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21544c6f-59af-4a59-a720-d40b64e76f5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:12.732029285Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 21544c6f-59af-4a59-a720-d40b64e76f5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:12.734079753Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ddb8e0b6-c193-419e-b893-def90acf2c6e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:12.820286905Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ddb8e0b6-c193-419e-b893-def90acf2c6e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:12.820342576Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:12.914247271Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:12.914307283Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:12.91454419Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:12.914688425Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:12.999674457Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:12.99977804Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:12.999791971Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:12.999805121Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:12.999816662Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_426e6991-0acc-4834-a7bc-68fb4dfbb775","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:13.093337443Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_ddb8e0b6-c193-419e-b893-def90acf2c6e","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:13.097759909Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_bf26395e-9671-4bee-b96d-1c6f76e85a02","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:13.102241666Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:13.106803026Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_38d5c382-d544-48a5-9e2e-b01d5cfb5bc9","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:13.111588073Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_cockroachdb_426e6991-0acc-4834-a7bc-68fb4dfbb775","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:13.116382011Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_ddb8e0b6-c193-419e-b893-def90acf2c6e","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:13.121201109Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_bf26395e-9671-4bee-b96d-1c6f76e85a02","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:13.12610039Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:13.130738212Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_38d5c382-d544-48a5-9e2e-b01d5cfb5bc9","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:13.135190698Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_cockroachdb_426e6991-0acc-4834-a7bc-68fb4dfbb775:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:16.058343769Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:16.058465413Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_426e6991-0acc-4834-a7bc-68fb4dfbb775","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:16.061588926Z","hostname":"buskin","pid":1671,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:776"} {"msg":"wait for service Some(\"oxz_cockroachdb_426e6991-0acc-4834-a7bc-68fb4dfbb775\") failed: Property not found. retry in 50.858442ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:16.198314787Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_426e6991-0acc-4834-a7bc-68fb4dfbb775","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:16.223393931Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:16.223528585Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_cockroachdb_426e6991-0acc-4834-a7bc-68fb4dfbb775\") failed: Property not found. retry in 53.096905ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:16.258611797Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_426e6991-0acc-4834-a7bc-68fb4dfbb775","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_cockroachdb_38d5c382-d544-48a5-9e2e-b01d5cfb5bc9:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:16.299055736Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_cockroachdb_bf26395e-9671-4bee-b96d-1c6f76e85a02:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:16.299118188Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:16.299129268Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_38d5c382-d544-48a5-9e2e-b01d5cfb5bc9","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:16.310087658Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_bf26395e-9671-4bee-b96d-1c6f76e85a02","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_cockroachdb_426e6991-0acc-4834-a7bc-68fb4dfbb775\") failed: Property not found. retry in 207.210987ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:16.331309625Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_426e6991-0acc-4834-a7bc-68fb4dfbb775","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_cockroachdb_ddb8e0b6-c193-419e-b893-def90acf2c6e:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:16.400977164Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:16.402606387Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_ddb8e0b6-c193-419e-b893-def90acf2c6e","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d\") failed: Property not found. retry in 69.759284ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:16.616965568Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_426e6991-0acc-4834-a7bc-68fb4dfbb775\") failed: Property not found. retry in 368.693575ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:16.787080536Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_426e6991-0acc-4834-a7bc-68fb4dfbb775","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d\") failed: Property not found. retry in 135.724081ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:16.89069937Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_38d5c382-d544-48a5-9e2e-b01d5cfb5bc9\") failed: Property not found. retry in 33.883864ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:16.909809628Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_38d5c382-d544-48a5-9e2e-b01d5cfb5bc9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bf26395e-9671-4bee-b96d-1c6f76e85a02\") failed: Property not found. retry in 38.386684ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:17.104056839Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_bf26395e-9671-4bee-b96d-1c6f76e85a02","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d\") failed: Property not found. retry in 132.444242ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:17.179128084Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_38d5c382-d544-48a5-9e2e-b01d5cfb5bc9\") failed: Property not found. retry in 55.589739ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:17.276052108Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_38d5c382-d544-48a5-9e2e-b01d5cfb5bc9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_426e6991-0acc-4834-a7bc-68fb4dfbb775\") failed: Property not found. retry in 679.184927ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:17.721498139Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_426e6991-0acc-4834-a7bc-68fb4dfbb775","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ddb8e0b6-c193-419e-b893-def90acf2c6e\") failed: Property not found. retry in 65.17629ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:17.793233676Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_ddb8e0b6-c193-419e-b893-def90acf2c6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bf26395e-9671-4bee-b96d-1c6f76e85a02\") failed: Property not found. retry in 98.758293ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:17.846980511Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_bf26395e-9671-4bee-b96d-1c6f76e85a02","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d\") failed: Property not found. retry in 258.95554ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:17.905637468Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_38d5c382-d544-48a5-9e2e-b01d5cfb5bc9\") failed: Property not found. retry in 149.758745ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:18.008409944Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_38d5c382-d544-48a5-9e2e-b01d5cfb5bc9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ddb8e0b6-c193-419e-b893-def90acf2c6e\") failed: Property not found. retry in 99.540397ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:18.085590629Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_ddb8e0b6-c193-419e-b893-def90acf2c6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bf26395e-9671-4bee-b96d-1c6f76e85a02\") failed: Property not found. retry in 102.555252ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:18.124153606Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_bf26395e-9671-4bee-b96d-1c6f76e85a02","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_38d5c382-d544-48a5-9e2e-b01d5cfb5bc9\") failed: Property not found. retry in 211.943096ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:18.189144181Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_38d5c382-d544-48a5-9e2e-b01d5cfb5bc9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ddb8e0b6-c193-419e-b893-def90acf2c6e\") failed: Property not found. retry in 150.771992ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:18.228718841Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_ddb8e0b6-c193-419e-b893-def90acf2c6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d\") failed: Property not found. retry in 421.58454ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:18.26431455Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bf26395e-9671-4bee-b96d-1c6f76e85a02\") failed: Property not found. retry in 244.320527ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:18.302403981Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_bf26395e-9671-4bee-b96d-1c6f76e85a02","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ddb8e0b6-c193-419e-b893-def90acf2c6e\") failed: Property not found. retry in 365.258562ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:18.410032526Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_ddb8e0b6-c193-419e-b893-def90acf2c6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_426e6991-0acc-4834-a7bc-68fb4dfbb775\") failed: Property not found. retry in 1.267956202s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:18.444196499Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_426e6991-0acc-4834-a7bc-68fb4dfbb775","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_38d5c382-d544-48a5-9e2e-b01d5cfb5bc9\") failed: Property not found. retry in 1.113169309s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:18.461672093Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_38d5c382-d544-48a5-9e2e-b01d5cfb5bc9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bf26395e-9671-4bee-b96d-1c6f76e85a02\") failed: Property not found. retry in 900.648927ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:18.659049766Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_bf26395e-9671-4bee-b96d-1c6f76e85a02","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d\") failed: Property not found. retry in 714.861548ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:18.735843079Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ddb8e0b6-c193-419e-b893-def90acf2c6e\") failed: Property not found. retry in 1.018716615s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:18.788124586Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_ddb8e0b6-c193-419e-b893-def90acf2c6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d\") failed: Property not found. retry in 726.271091ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:19.459791639Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bf26395e-9671-4bee-b96d-1c6f76e85a02\") failed: Property not found. retry in 571.608586ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:19.567653432Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_bf26395e-9671-4bee-b96d-1c6f76e85a02","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_38d5c382-d544-48a5-9e2e-b01d5cfb5bc9\") failed: Property not found. retry in 1.088628355s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:19.584144353Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_38d5c382-d544-48a5-9e2e-b01d5cfb5bc9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_426e6991-0acc-4834-a7bc-68fb4dfbb775\") failed: Property not found. retry in 1.132334945s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:19.721389262Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_426e6991-0acc-4834-a7bc-68fb4dfbb775","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ddb8e0b6-c193-419e-b893-def90acf2c6e\") failed: Property not found. retry in 854.671168ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:19.815311467Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_ddb8e0b6-c193-419e-b893-def90acf2c6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bf26395e-9671-4bee-b96d-1c6f76e85a02\") failed: Property not found. retry in 1.133019301s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:20.148579074Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_bf26395e-9671-4bee-b96d-1c6f76e85a02","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d\") failed: Property not found. retry in 630.230152ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:20.196356544Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ddb8e0b6-c193-419e-b893-def90acf2c6e\") failed: Property not found. retry in 586.196004ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:20.680163946Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_ddb8e0b6-c193-419e-b893-def90acf2c6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_38d5c382-d544-48a5-9e2e-b01d5cfb5bc9\") failed: Property not found. retry in 1.323725223s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:20.688036144Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_38d5c382-d544-48a5-9e2e-b01d5cfb5bc9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d\") failed: Property not found. retry in 1.261157232s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:20.834417242Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_426e6991-0acc-4834-a7bc-68fb4dfbb775\") failed: Property not found. retry in 1.305156346s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:20.862227186Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_426e6991-0acc-4834-a7bc-68fb4dfbb775","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ddb8e0b6-c193-419e-b893-def90acf2c6e\") failed: Property not found. retry in 1.295022008s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:21.275209652Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_ddb8e0b6-c193-419e-b893-def90acf2c6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bf26395e-9671-4bee-b96d-1c6f76e85a02\") failed: Property not found. retry in 699.623304ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:21.289659246Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_bf26395e-9671-4bee-b96d-1c6f76e85a02","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bf26395e-9671-4bee-b96d-1c6f76e85a02\") failed: Property not found. retry in 823.408277ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:21.998761059Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_bf26395e-9671-4bee-b96d-1c6f76e85a02","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_38d5c382-d544-48a5-9e2e-b01d5cfb5bc9\") failed: Property not found. retry in 1.419338205s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:22.020340968Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_38d5c382-d544-48a5-9e2e-b01d5cfb5bc9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d\") failed: Property not found. retry in 1.302541124s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:22.104184452Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_426e6991-0acc-4834-a7bc-68fb4dfbb775\") failed: Property not found. retry in 1.409499165s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:22.175937169Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_426e6991-0acc-4834-a7bc-68fb4dfbb775","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ddb8e0b6-c193-419e-b893-def90acf2c6e\") failed: Property not found. retry in 1.273725946s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:22.578966387Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_ddb8e0b6-c193-419e-b893-def90acf2c6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bf26395e-9671-4bee-b96d-1c6f76e85a02\") failed: Property not found. retry in 928.659582ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:22.831544504Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_bf26395e-9671-4bee-b96d-1c6f76e85a02","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d\") failed: Property not found. retry in 1.288959633s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:23.415447964Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_38d5c382-d544-48a5-9e2e-b01d5cfb5bc9\") failed: Property not found. retry in 881.515691ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:23.449134771Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_38d5c382-d544-48a5-9e2e-b01d5cfb5bc9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_426e6991-0acc-4834-a7bc-68fb4dfbb775\") failed: Property not found. retry in 1.483708602s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:23.59431456Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_426e6991-0acc-4834-a7bc-68fb4dfbb775","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bf26395e-9671-4bee-b96d-1c6f76e85a02\") failed: Property not found. retry in 1.058644332s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:23.76907334Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_bf26395e-9671-4bee-b96d-1c6f76e85a02","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ddb8e0b6-c193-419e-b893-def90acf2c6e\") failed: Property not found. retry in 825.564571ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:23.86069116Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_ddb8e0b6-c193-419e-b893-def90acf2c6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_38d5c382-d544-48a5-9e2e-b01d5cfb5bc9\") failed: Property not found. retry in 1.070858347s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:24.339148386Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_38d5c382-d544-48a5-9e2e-b01d5cfb5bc9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:24.386207432Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:58695"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:24.386413149Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"30b9446b-3dff-448b-81bb-c1b50cd6ed45","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:58695","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":116,"response_code":"200"} {"msg":"wait for service Some(\"oxz_cockroachdb_ddb8e0b6-c193-419e-b893-def90acf2c6e\") failed: Property not found. retry in 584.80107ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:24.695268224Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_ddb8e0b6-c193-419e-b893-def90acf2c6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d\") failed: Property not found. retry in 1.121282139s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:24.713868235Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bf26395e-9671-4bee-b96d-1c6f76e85a02\") failed: Property not found. retry in 500.980849ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:24.835049765Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_bf26395e-9671-4bee-b96d-1c6f76e85a02","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_426e6991-0acc-4834-a7bc-68fb4dfbb775\") failed: Property not found. retry in 766.565607ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:25.089327568Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_426e6991-0acc-4834-a7bc-68fb4dfbb775","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ddb8e0b6-c193-419e-b893-def90acf2c6e\") failed: Property not found. retry in 649.148122ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:25.294008971Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_ddb8e0b6-c193-419e-b893-def90acf2c6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bf26395e-9671-4bee-b96d-1c6f76e85a02\") failed: Property not found. retry in 738.183799ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:25.348572054Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_bf26395e-9671-4bee-b96d-1c6f76e85a02","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_38d5c382-d544-48a5-9e2e-b01d5cfb5bc9\") failed: Property not found. retry in 1.309064625s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:25.420752225Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_38d5c382-d544-48a5-9e2e-b01d5cfb5bc9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d\") failed: Property not found. retry in 623.256046ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:25.845575249Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_426e6991-0acc-4834-a7bc-68fb4dfbb775\") failed: Property not found. retry in 1.292085063s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:25.864893074Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_426e6991-0acc-4834-a7bc-68fb4dfbb775","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ddb8e0b6-c193-419e-b893-def90acf2c6e\") failed: Property not found. retry in 560.858473ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:25.953701481Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_ddb8e0b6-c193-419e-b893-def90acf2c6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bf26395e-9671-4bee-b96d-1c6f76e85a02\") failed: Property not found. retry in 1.060342648s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:26.097501244Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_bf26395e-9671-4bee-b96d-1c6f76e85a02","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d\") failed: Property not found. retry in 747.817142ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:26.478545461Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ddb8e0b6-c193-419e-b893-def90acf2c6e\") failed: Property not found. retry in 1.089345125s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:26.525944208Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_ddb8e0b6-c193-419e-b893-def90acf2c6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_38d5c382-d544-48a5-9e2e-b01d5cfb5bc9\") failed: Property not found. retry in 1.475449633s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:26.739890086Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_38d5c382-d544-48a5-9e2e-b01d5cfb5bc9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_426e6991-0acc-4834-a7bc-68fb4dfbb775\") failed: Property not found. retry in 590.454575ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:27.166005873Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_426e6991-0acc-4834-a7bc-68fb4dfbb775","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bf26395e-9671-4bee-b96d-1c6f76e85a02\") failed: Property not found. retry in 642.111428ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:27.172809596Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_bf26395e-9671-4bee-b96d-1c6f76e85a02","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d\") failed: Property not found. retry in 1.289031298s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:27.234970498Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ddb8e0b6-c193-419e-b893-def90acf2c6e\") failed: Property not found. retry in 780.840228ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:27.623764129Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_ddb8e0b6-c193-419e-b893-def90acf2c6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_426e6991-0acc-4834-a7bc-68fb4dfbb775\") failed: Property not found. retry in 842.489722ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:27.765322909Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_426e6991-0acc-4834-a7bc-68fb4dfbb775","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bf26395e-9671-4bee-b96d-1c6f76e85a02\") failed: Property not found. retry in 1.166901487s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:27.824178682Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_bf26395e-9671-4bee-b96d-1c6f76e85a02","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_38d5c382-d544-48a5-9e2e-b01d5cfb5bc9\") failed: Property not found. retry in 992.575424ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:28.225113662Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_38d5c382-d544-48a5-9e2e-b01d5cfb5bc9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ddb8e0b6-c193-419e-b893-def90acf2c6e\") failed: Property not found. retry in 1.4638946s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:28.413542282Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_ddb8e0b6-c193-419e-b893-def90acf2c6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d\") failed: Property not found. retry in 1.483485105s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:28.534865896Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_426e6991-0acc-4834-a7bc-68fb4dfbb775\") failed: Property not found. retry in 508.937445ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:28.616282831Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_426e6991-0acc-4834-a7bc-68fb4dfbb775","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bf26395e-9671-4bee-b96d-1c6f76e85a02\") failed: Property not found. retry in 1.240594926s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:29.001142573Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_bf26395e-9671-4bee-b96d-1c6f76e85a02","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_426e6991-0acc-4834-a7bc-68fb4dfbb775\") failed: Property not found. retry in 887.489752ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:29.134866815Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_426e6991-0acc-4834-a7bc-68fb4dfbb775","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_38d5c382-d544-48a5-9e2e-b01d5cfb5bc9\") failed: Property not found. retry in 1.338414343s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:29.226551447Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_38d5c382-d544-48a5-9e2e-b01d5cfb5bc9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ddb8e0b6-c193-419e-b893-def90acf2c6e\") failed: Property not found. retry in 1.121516696s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:29.887995544Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_ddb8e0b6-c193-419e-b893-def90acf2c6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d\") failed: Property not found. retry in 710.088386ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:30.027885699Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_426e6991-0acc-4834-a7bc-68fb4dfbb775\") failed: Property not found. retry in 1.288239586s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:30.037807355Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_426e6991-0acc-4834-a7bc-68fb4dfbb775","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bf26395e-9671-4bee-b96d-1c6f76e85a02\") failed: Property not found. retry in 1.054923314s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:30.250096428Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_bf26395e-9671-4bee-b96d-1c6f76e85a02","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d\") failed: Property not found. retry in 1.391543082s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:30.74819138Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d\") failed: Property not found. retry in 788.721801ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:32.149099587Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d\") failed: Property not found. retry in 1.208661261s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:32.947144191Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d\") failed: Property not found. retry in 1.076503657s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:34.16606214Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d\") failed: Property not found. retry in 1.046769982s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:35.633713279Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d\") failed: Property not found. retry in 1.103836977s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:36.692345413Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d\") failed: Property not found. retry in 720.631487ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:37.8067929Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d\") failed: Property not found. retry in 703.771345ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:38.536628643Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d\") failed: Property not found. retry in 539.664154ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:39.250757661Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:40.533580827Z","hostname":"buskin","pid":1671,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:776"} {"msg":"wait for service Some(\"oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d\") failed: Property not found. retry in 1.067496099s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:40.541032752Z","hostname":"buskin","pid":1671,"zone":"oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:41.6284323Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_cockroachdb_426e6991-0acc-4834-a7bc-68fb4dfbb775"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:41.628517013Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_cockroachdb_ddb8e0b6-c193-419e-b893-def90acf2c6e"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:41.628528673Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_cockroachdb_bf26395e-9671-4bee-b96d-1c6f76e85a02"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:41.628538044Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:41.628548444Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_cockroachdb_38d5c382-d544-48a5-9e2e-b01d5cfb5bc9"} {"msg":"Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:41.628590295Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:198"} {"msg":"Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:41.629242697Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:198"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:41.630087764Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"eb034644-d4a9-4792-8db6-6bd0bdb8b2ea","remote_addr":"[fd00:1122:3344:101::1]:61010","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":29160085,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:41.663762141Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:63762"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:41.695437861Z","hostname":"buskin","pid":1671,"uri":"/cockroachdb","method":"POST","req_id":"6060e02e-4d0e-416a-a9ae-c4f78ba7b52c","remote_addr":"[fd00:1122:3344:101::1]:63762","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService8/omicron6 in oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d /usr/sbin/ipadm show-addr -p -o ADDR oxControlService8/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":31510,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:41.695753922Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:755","retry_after":"110.35299ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6060e02e-4d0e-416a-a9ae-c4f78ba7b52c\", \"content-length\": \"124\", \"date\": \"Thu, 15 Feb 2024 05:56:41 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6060e02e-4d0e-416a-a9ae-c4f78ba7b52c\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:41.835815122Z","hostname":"buskin","pid":1671,"uri":"/cockroachdb","method":"POST","req_id":"50aeb35e-71dc-4e75-9f0b-44e81576d1bf","remote_addr":"[fd00:1122:3344:101::1]:63762","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService8/omicron6 in oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d /usr/sbin/ipadm show-addr -p -o ADDR oxControlService8/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":27852,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:41.836134893Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:755","retry_after":"151.552711ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"50aeb35e-71dc-4e75-9f0b-44e81576d1bf\", \"content-length\": \"124\", \"date\": \"Thu, 15 Feb 2024 05:56:41 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"50aeb35e-71dc-4e75-9f0b-44e81576d1bf\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:42.01654929Z","hostname":"buskin","pid":1671,"uri":"/cockroachdb","method":"POST","req_id":"aa8aa570-289f-4355-9c61-435569e075d4","remote_addr":"[fd00:1122:3344:101::1]:63762","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService8/omicron6 in oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d /usr/sbin/ipadm show-addr -p -o ADDR oxControlService8/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":28147,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:42.016751397Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:755","retry_after":"198.651899ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"aa8aa570-289f-4355-9c61-435569e075d4\", \"content-length\": \"124\", \"date\": \"Thu, 15 Feb 2024 05:56:41 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"aa8aa570-289f-4355-9c61-435569e075d4\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:42.244128146Z","hostname":"buskin","pid":1671,"uri":"/cockroachdb","method":"POST","req_id":"8dc359ec-6c25-4f87-b45a-60727d5c6c44","remote_addr":"[fd00:1122:3344:101::1]:63762","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService8/omicron6 in oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d /usr/sbin/ipadm show-addr -p -o ADDR oxControlService8/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":27761,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:42.244323842Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:755","retry_after":"116.640915ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8dc359ec-6c25-4f87-b45a-60727d5c6c44\", \"content-length\": \"124\", \"date\": \"Thu, 15 Feb 2024 05:56:41 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"8dc359ec-6c25-4f87-b45a-60727d5c6c44\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:42.387923872Z","hostname":"buskin","pid":1671,"uri":"/cockroachdb","method":"POST","req_id":"1f00201f-8981-4299-ac8f-93ce2bd0f7d3","remote_addr":"[fd00:1122:3344:101::1]:63762","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService8/omicron6 in oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d /usr/sbin/ipadm show-addr -p -o ADDR oxControlService8/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":26763,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:42.388127379Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:755","retry_after":"304.822271ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1f00201f-8981-4299-ac8f-93ce2bd0f7d3\", \"content-length\": \"124\", \"date\": \"Thu, 15 Feb 2024 05:56:41 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"1f00201f-8981-4299-ac8f-93ce2bd0f7d3\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:42.720354982Z","hostname":"buskin","pid":1671,"uri":"/cockroachdb","method":"POST","req_id":"62776f4c-32c7-4523-9ced-f1b635ad2bea","remote_addr":"[fd00:1122:3344:101::1]:63762","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService8/omicron6 in oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d /usr/sbin/ipadm show-addr -p -o ADDR oxControlService8/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":26830,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:42.720567269Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:755","retry_after":"324.004329ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"62776f4c-32c7-4523-9ced-f1b635ad2bea\", \"content-length\": \"124\", \"date\": \"Thu, 15 Feb 2024 05:56:42 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"62776f4c-32c7-4523-9ced-f1b635ad2bea\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:43.072302624Z","hostname":"buskin","pid":1671,"uri":"/cockroachdb","method":"POST","req_id":"506b05d7-e8a6-4958-b8b8-b731dd5f35f4","remote_addr":"[fd00:1122:3344:101::1]:63762","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService8/omicron6 in oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d /usr/sbin/ipadm show-addr -p -o ADDR oxControlService8/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":27474,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:43.072514571Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:755","retry_after":"324.777442ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"506b05d7-e8a6-4958-b8b8-b731dd5f35f4\", \"content-length\": \"124\", \"date\": \"Thu, 15 Feb 2024 05:56:42 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"506b05d7-e8a6-4958-b8b8-b731dd5f35f4\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:43.423654436Z","hostname":"buskin","pid":1671,"uri":"/cockroachdb","method":"POST","req_id":"e94bc4a7-acf3-47f3-9d22-718d1b539136","remote_addr":"[fd00:1122:3344:101::1]:63762","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService8/omicron6 in oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_21544c6f-59af-4a59-a720-d40b64e76f5d /usr/sbin/ipadm show-addr -p -o ADDR oxControlService8/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":25792,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:43.423908015Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:755","retry_after":"493.737716ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e94bc4a7-acf3-47f3-9d22-718d1b539136\", \"content-length\": \"124\", \"date\": \"Thu, 15 Feb 2024 05:56:42 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e94bc4a7-acf3-47f3-9d22-718d1b539136\" }"} {"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::6]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:43.943563231Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3165"} {"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:50.933837169Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3183"} {"msg":"Formatting CRDB - Completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:57.645284263Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:3206"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:57.645401117Z","hostname":"buskin","pid":1671,"uri":"/cockroachdb","method":"POST","req_id":"3a6fe5f6-2fac-4417-a0f3-cfcc15d8135e","remote_addr":"[fd00:1122:3344:101::1]:63762","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":13765855,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:57.645594063Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:60964"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:57.646842054Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"3075da24-3378-4695-b2e7-daace8fdb229","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:60964","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":1013,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 719635e7-d366-4895-9272-9ade9dc77ba8, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2ae10e87-0030-442b-9981-0b4c70a9c369, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a62bfe46-059b-4ea1-b2a5-7f98d258138d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 44034536-a7b3-4ef5-9ea6-8cd61921d1ee, kind: Service { id: a62bfe46-059b-4ea1-b2a5-7f98d258138d }, name: Name(\"ntp-a62bfe46-059b-4ea1-b2a5-7f98d258138d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 159])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bf26395e-9671-4bee-b96d-1c6f76e85a02, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 21544c6f-59af-4a59-a720-d40b64e76f5d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: ddb8e0b6-c193-419e-b893-def90acf2c6e, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: a702a2df-8aec-4a4e-9585-52df0fbccf45, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 80b71d9a-f960-4a96-9d83-f9c1c3b2663c, kind: Service { id: a702a2df-8aec-4a4e-9585-52df0fbccf45 }, name: Name(\"external-dns-a702a2df-8aec-4a4e-9585-52df0fbccf45\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 15334e02-b404-488e-9ce7-2a50df738630, kind: Service { id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39 }, name: Name(\"external-dns-7e2aceaf-08e1-46f2-95ec-f9032200ba39\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 88])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 28701ada-e899-402b-a2fd-f7a3fd7c41df, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 0e8e6699-5046-483c-bebf-fffb211be035, kind: Service { id: 28701ada-e899-402b-a2fd-f7a3fd7c41df }, name: Name(\"nexus-28701ada-e899-402b-a2fd-f7a3fd7c41df\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9363e23e-d709-4bd9-bca4-7fbe3ad55235, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 9e4a840d-8987-4208-af8d-b4a2771e9cc4, kind: Service { id: 9363e23e-d709-4bd9-bca4-7fbe3ad55235 }, name: Name(\"nexus-9363e23e-d709-4bd9-bca4-7fbe3ad55235\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 232, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4243c086-1bed-4413-b0c7-7c802e7bc575, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: a0dd5105-fcbd-4055-ae12-c3fb0abcf0fa, kind: Service { id: 4243c086-1bed-4413-b0c7-7c802e7bc575 }, name: Name(\"nexus-4243c086-1bed-4413-b0c7-7c802e7bc575\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 15])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 07eefccc-a16d-41be-9df2-0faa4294ae1c, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a0f87398-93b9-40a8-90b6-7059441c2401, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 2cb47b44-e1c8-4597-82f0-7ee9d5d86893, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 0e3a6bd4-4538-4ccd-a063-9c7342f19203, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f7332e25-0a75-4959-b498-faa862ee2c5f, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: d6729570-1e52-4d84-af84-f4f23623dfb6, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 1d816ece-0442-4979-9d32-7665ef85afde, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 98f9db1c-1595-4f22-a928-321363864abe, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:57.680099109Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:57.680505113Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:35849"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:57.680828964Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:57.680837434Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 719635e7-d366-4895-9272-9ade9dc77ba8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:57.702361398Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 719635e7-d366-4895-9272-9ade9dc77ba8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:57.702416584Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ae10e87-0030-442b-9981-0b4c70a9c369, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:57.722077352Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2ae10e87-0030-442b-9981-0b4c70a9c369, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:57.722133164Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:57.742177704Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:57.742223046Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bf26395e-9671-4bee-b96d-1c6f76e85a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:57.760353733Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bf26395e-9671-4bee-b96d-1c6f76e85a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:57.760405394Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:57.777986704Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:57.780247828Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21544c6f-59af-4a59-a720-d40b64e76f5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:57.795418738Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 21544c6f-59af-4a59-a720-d40b64e76f5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:57.795470449Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ddb8e0b6-c193-419e-b893-def90acf2c6e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:57.812592753Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ddb8e0b6-c193-419e-b893-def90acf2c6e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:57.812642235Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a702a2df-8aec-4a4e-9585-52df0fbccf45, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:57.831355541Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a702a2df-8aec-4a4e-9585-52df0fbccf45, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:57.831401983Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:57.936771594Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:57.936823755Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:58.065232854Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:58.065299576Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:58.150686798Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:58.15074646Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:58.252978808Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:58.25305136Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:58.400746745Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:58.400807977Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7332e25-0a75-4959-b498-faa862ee2c5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:58.503397976Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f7332e25-0a75-4959-b498-faa862ee2c5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:58.503464388Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6729570-1e52-4d84-af84-f4f23623dfb6, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:58.622774208Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d6729570-1e52-4d84-af84-f4f23623dfb6, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:58.622821849Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1d816ece-0442-4979-9d32-7665ef85afde, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:58.738109616Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1d816ece-0442-4979-9d32-7665ef85afde, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:58.738180479Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 98f9db1c-1595-4f22-a928-321363864abe, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:58.850254515Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 98f9db1c-1595-4f22-a928-321363864abe, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:58.850313042Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:58.959209999Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:58.959272611Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:59.102287281Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:59.102341313Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:59.203512705Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:59.203576318Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:59.203846746Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:59.204019072Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:59.317404097Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:59.317483529Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:59.31749396Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:59.3175033Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:59.31751214Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:59.31752084Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:59.317529551Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:59.317536661Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:59.317543681Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:59.317552521Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:59.317569562Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:59.317577912Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:59.317586203Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:59.317596033Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:59.317604603Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:59.317613294Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:59.317620244Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:59.317628494Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:59.317637374Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:59.317748278Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"3cb793ac-29ac-4ef0-a973-45113fd92e79","remote_addr":"[fd00:1122:3344:101::1]:35849","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302\", TimeNotSynchronized), (\"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f\", TimeNotSynchronized), (\"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde\", TimeNotSynchronized), (\"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235\", TimeNotSynchronized), (\"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575\", TimeNotSynchronized), (\"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6\", TimeNotSynchronized), (\"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7\", TimeNotSynchronized), (\"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c\", TimeNotSynchronized), (\"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23\", TimeNotSynchronized), (\"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55\", TimeNotSynchronized), (\"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17\", TimeNotSynchronized), (\"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2\", TimeNotSynchronized), (\"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df\", TimeNotSynchronized), (\"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893\", TimeNotSynchronized), (\"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45\", TimeNotSynchronized), (\"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203\", TimeNotSynchronized), (\"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401\", TimeNotSynchronized), (\"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe\", TimeNotSynchronized), (\"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39\", TimeNotSynchronized)]","latency_us":1641612,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:59.318017077Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"74.479187ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3cb793ac-29ac-4ef0-a973-45113fd92e79\", \"content-length\": \"124\", \"date\": \"Thu, 15 Feb 2024 05:56:59 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3cb793ac-29ac-4ef0-a973-45113fd92e79\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 719635e7-d366-4895-9272-9ade9dc77ba8, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2ae10e87-0030-442b-9981-0b4c70a9c369, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a62bfe46-059b-4ea1-b2a5-7f98d258138d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 44034536-a7b3-4ef5-9ea6-8cd61921d1ee, kind: Service { id: a62bfe46-059b-4ea1-b2a5-7f98d258138d }, name: Name(\"ntp-a62bfe46-059b-4ea1-b2a5-7f98d258138d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 159])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bf26395e-9671-4bee-b96d-1c6f76e85a02, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 21544c6f-59af-4a59-a720-d40b64e76f5d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: ddb8e0b6-c193-419e-b893-def90acf2c6e, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: a702a2df-8aec-4a4e-9585-52df0fbccf45, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 80b71d9a-f960-4a96-9d83-f9c1c3b2663c, kind: Service { id: a702a2df-8aec-4a4e-9585-52df0fbccf45 }, name: Name(\"external-dns-a702a2df-8aec-4a4e-9585-52df0fbccf45\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 15334e02-b404-488e-9ce7-2a50df738630, kind: Service { id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39 }, name: Name(\"external-dns-7e2aceaf-08e1-46f2-95ec-f9032200ba39\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 88])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 28701ada-e899-402b-a2fd-f7a3fd7c41df, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 0e8e6699-5046-483c-bebf-fffb211be035, kind: Service { id: 28701ada-e899-402b-a2fd-f7a3fd7c41df }, name: Name(\"nexus-28701ada-e899-402b-a2fd-f7a3fd7c41df\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9363e23e-d709-4bd9-bca4-7fbe3ad55235, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 9e4a840d-8987-4208-af8d-b4a2771e9cc4, kind: Service { id: 9363e23e-d709-4bd9-bca4-7fbe3ad55235 }, name: Name(\"nexus-9363e23e-d709-4bd9-bca4-7fbe3ad55235\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 232, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4243c086-1bed-4413-b0c7-7c802e7bc575, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: a0dd5105-fcbd-4055-ae12-c3fb0abcf0fa, kind: Service { id: 4243c086-1bed-4413-b0c7-7c802e7bc575 }, name: Name(\"nexus-4243c086-1bed-4413-b0c7-7c802e7bc575\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 15])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 07eefccc-a16d-41be-9df2-0faa4294ae1c, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a0f87398-93b9-40a8-90b6-7059441c2401, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 2cb47b44-e1c8-4597-82f0-7ee9d5d86893, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 0e3a6bd4-4538-4ccd-a063-9c7342f19203, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f7332e25-0a75-4959-b498-faa862ee2c5f, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: d6729570-1e52-4d84-af84-f4f23623dfb6, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 1d816ece-0442-4979-9d32-7665ef85afde, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 98f9db1c-1595-4f22-a928-321363864abe, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:59.392937904Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:59.393577536Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:59.393592736Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 719635e7-d366-4895-9272-9ade9dc77ba8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:59.414077701Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 719635e7-d366-4895-9272-9ade9dc77ba8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:59.414130683Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ae10e87-0030-442b-9981-0b4c70a9c369, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:59.433635575Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2ae10e87-0030-442b-9981-0b4c70a9c369, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:59.433691837Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:59.452216857Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:59.452267009Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bf26395e-9671-4bee-b96d-1c6f76e85a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:59.471455341Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bf26395e-9671-4bee-b96d-1c6f76e85a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:59.471516363Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:59.489610389Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:59.48965934Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21544c6f-59af-4a59-a720-d40b64e76f5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:59.507397405Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 21544c6f-59af-4a59-a720-d40b64e76f5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:59.507444505Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ddb8e0b6-c193-419e-b893-def90acf2c6e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:59.525730789Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ddb8e0b6-c193-419e-b893-def90acf2c6e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:59.52578261Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a702a2df-8aec-4a4e-9585-52df0fbccf45, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:59.54339119Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a702a2df-8aec-4a4e-9585-52df0fbccf45, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:59.543451512Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:59.561756495Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:59.561806197Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:59.58041815Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:59.580468061Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:59.598532396Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:59.598578158Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:59.616471477Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:59.616525999Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:59.633727215Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:59.633774777Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7332e25-0a75-4959-b498-faa862ee2c5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:59.651693847Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f7332e25-0a75-4959-b498-faa862ee2c5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:59.651746569Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6729570-1e52-4d84-af84-f4f23623dfb6, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:59.669629088Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d6729570-1e52-4d84-af84-f4f23623dfb6, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:59.66968396Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1d816ece-0442-4979-9d32-7665ef85afde, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:59.687463265Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1d816ece-0442-4979-9d32-7665ef85afde, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:59.687513867Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 98f9db1c-1595-4f22-a928-321363864abe, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:59.70520521Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 98f9db1c-1595-4f22-a928-321363864abe, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:59.705255871Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:59.723091299Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:59.723143851Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:59.741085422Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:59.741161734Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:59.760882904Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:59.760944076Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:59.761180913Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:59.761317528Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:59.853885367Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:59.853952369Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:59.8539625Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:59.85397197Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:59.85397897Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:59.85398778Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:59.853994781Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:59.854012711Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:59.854021502Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:59.854030592Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:59.854039232Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:59.854047972Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:59.854054863Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:59.854062833Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:59.854071523Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:59.854078083Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:59.854085884Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:59.854092564Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:59.854099034Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:59.854189107Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"7fc64f05-a76a-43a7-9298-5aad6dece151","remote_addr":"[fd00:1122:3344:101::1]:35849","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235\", TimeNotSynchronized), (\"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302\", TimeNotSynchronized), (\"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2\", TimeNotSynchronized), (\"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893\", TimeNotSynchronized), (\"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23\", TimeNotSynchronized), (\"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55\", TimeNotSynchronized), (\"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df\", TimeNotSynchronized), (\"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c\", TimeNotSynchronized), (\"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45\", TimeNotSynchronized), (\"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401\", TimeNotSynchronized), (\"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17\", TimeNotSynchronized), (\"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7\", TimeNotSynchronized), (\"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39\", TimeNotSynchronized), (\"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde\", TimeNotSynchronized), (\"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe\", TimeNotSynchronized), (\"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575\", TimeNotSynchronized), (\"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6\", TimeNotSynchronized), (\"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f\", TimeNotSynchronized), (\"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203\", TimeNotSynchronized)]","latency_us":462062,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:56:59.854411564Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"68.897422ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7fc64f05-a76a-43a7-9298-5aad6dece151\", \"content-length\": \"124\", \"date\": \"Thu, 15 Feb 2024 05:56:59 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"7fc64f05-a76a-43a7-9298-5aad6dece151\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 719635e7-d366-4895-9272-9ade9dc77ba8, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2ae10e87-0030-442b-9981-0b4c70a9c369, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a62bfe46-059b-4ea1-b2a5-7f98d258138d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 44034536-a7b3-4ef5-9ea6-8cd61921d1ee, kind: Service { id: a62bfe46-059b-4ea1-b2a5-7f98d258138d }, name: Name(\"ntp-a62bfe46-059b-4ea1-b2a5-7f98d258138d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 159])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bf26395e-9671-4bee-b96d-1c6f76e85a02, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 21544c6f-59af-4a59-a720-d40b64e76f5d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: ddb8e0b6-c193-419e-b893-def90acf2c6e, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: a702a2df-8aec-4a4e-9585-52df0fbccf45, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 80b71d9a-f960-4a96-9d83-f9c1c3b2663c, kind: Service { id: a702a2df-8aec-4a4e-9585-52df0fbccf45 }, name: Name(\"external-dns-a702a2df-8aec-4a4e-9585-52df0fbccf45\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 15334e02-b404-488e-9ce7-2a50df738630, kind: Service { id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39 }, name: Name(\"external-dns-7e2aceaf-08e1-46f2-95ec-f9032200ba39\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 88])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 28701ada-e899-402b-a2fd-f7a3fd7c41df, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 0e8e6699-5046-483c-bebf-fffb211be035, kind: Service { id: 28701ada-e899-402b-a2fd-f7a3fd7c41df }, name: Name(\"nexus-28701ada-e899-402b-a2fd-f7a3fd7c41df\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9363e23e-d709-4bd9-bca4-7fbe3ad55235, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 9e4a840d-8987-4208-af8d-b4a2771e9cc4, kind: Service { id: 9363e23e-d709-4bd9-bca4-7fbe3ad55235 }, name: Name(\"nexus-9363e23e-d709-4bd9-bca4-7fbe3ad55235\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 232, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4243c086-1bed-4413-b0c7-7c802e7bc575, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: a0dd5105-fcbd-4055-ae12-c3fb0abcf0fa, kind: Service { id: 4243c086-1bed-4413-b0c7-7c802e7bc575 }, name: Name(\"nexus-4243c086-1bed-4413-b0c7-7c802e7bc575\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 15])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 07eefccc-a16d-41be-9df2-0faa4294ae1c, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a0f87398-93b9-40a8-90b6-7059441c2401, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 2cb47b44-e1c8-4597-82f0-7ee9d5d86893, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 0e3a6bd4-4538-4ccd-a063-9c7342f19203, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f7332e25-0a75-4959-b498-faa862ee2c5f, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: d6729570-1e52-4d84-af84-f4f23623dfb6, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 1d816ece-0442-4979-9d32-7665ef85afde, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 98f9db1c-1595-4f22-a928-321363864abe, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:59.923415797Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:59.924614857Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:59.924630497Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 719635e7-d366-4895-9272-9ade9dc77ba8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:59.944657507Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 719635e7-d366-4895-9272-9ade9dc77ba8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:59.944721259Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ae10e87-0030-442b-9981-0b4c70a9c369, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:59.963740895Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2ae10e87-0030-442b-9981-0b4c70a9c369, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:59.963798687Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:59.982914667Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:56:59.982967129Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bf26395e-9671-4bee-b96d-1c6f76e85a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.002166371Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bf26395e-9671-4bee-b96d-1c6f76e85a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.002220723Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.023716941Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.023770202Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21544c6f-59af-4a59-a720-d40b64e76f5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.042970375Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 21544c6f-59af-4a59-a720-d40b64e76f5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.043019536Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ddb8e0b6-c193-419e-b893-def90acf2c6e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.062415825Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ddb8e0b6-c193-419e-b893-def90acf2c6e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.062475387Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a702a2df-8aec-4a4e-9585-52df0fbccf45, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.082154365Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a702a2df-8aec-4a4e-9585-52df0fbccf45, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.082207887Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.10234432Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.102396172Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.121474331Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.121527812Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.141151969Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.141219161Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.159137851Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.159186733Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.177475175Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.177531277Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7332e25-0a75-4959-b498-faa862ee2c5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.196068357Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f7332e25-0a75-4959-b498-faa862ee2c5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.196120959Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6729570-1e52-4d84-af84-f4f23623dfb6, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.214586847Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d6729570-1e52-4d84-af84-f4f23623dfb6, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.214646689Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1d816ece-0442-4979-9d32-7665ef85afde, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.232831468Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1d816ece-0442-4979-9d32-7665ef85afde, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.23288567Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 98f9db1c-1595-4f22-a928-321363864abe, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.251430128Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 98f9db1c-1595-4f22-a928-321363864abe, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.251477622Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.269376577Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.271665878Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.2881394Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.288194662Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.306592768Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.3066537Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.306869247Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.307002261Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:00.401766223Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:00.401817124Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:00.401825545Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:00.401830545Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:00.401836775Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:00.401849365Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:00.401855655Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:00.401861636Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:00.401867186Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:00.401872756Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:00.401878486Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:00.401883986Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:00.401888147Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:00.401893787Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:00.401898377Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:00.401905807Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:00.401913547Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:00.401919528Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:00.401926968Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.402058242Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"ddb6b96f-44e1-424e-91c0-22e1c341e705","remote_addr":"[fd00:1122:3344:101::1]:35849","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235\", TimeNotSynchronized), (\"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe\", TimeNotSynchronized), (\"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55\", TimeNotSynchronized), (\"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575\", TimeNotSynchronized), (\"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203\", TimeNotSynchronized), (\"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39\", TimeNotSynchronized), (\"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df\", TimeNotSynchronized), (\"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2\", TimeNotSynchronized), (\"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401\", TimeNotSynchronized), (\"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f\", TimeNotSynchronized), (\"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde\", TimeNotSynchronized), (\"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17\", TimeNotSynchronized), (\"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23\", TimeNotSynchronized), (\"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c\", TimeNotSynchronized), (\"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6\", TimeNotSynchronized), (\"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302\", TimeNotSynchronized), (\"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893\", TimeNotSynchronized), (\"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7\", TimeNotSynchronized), (\"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45\", TimeNotSynchronized)]","latency_us":479530,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:00.40228548Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"185.992213ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ddb6b96f-44e1-424e-91c0-22e1c341e705\", \"content-length\": \"124\", \"date\": \"Thu, 15 Feb 2024 05:57:00 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ddb6b96f-44e1-424e-91c0-22e1c341e705\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 719635e7-d366-4895-9272-9ade9dc77ba8, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2ae10e87-0030-442b-9981-0b4c70a9c369, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a62bfe46-059b-4ea1-b2a5-7f98d258138d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 44034536-a7b3-4ef5-9ea6-8cd61921d1ee, kind: Service { id: a62bfe46-059b-4ea1-b2a5-7f98d258138d }, name: Name(\"ntp-a62bfe46-059b-4ea1-b2a5-7f98d258138d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 159])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bf26395e-9671-4bee-b96d-1c6f76e85a02, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 21544c6f-59af-4a59-a720-d40b64e76f5d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: ddb8e0b6-c193-419e-b893-def90acf2c6e, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: a702a2df-8aec-4a4e-9585-52df0fbccf45, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 80b71d9a-f960-4a96-9d83-f9c1c3b2663c, kind: Service { id: a702a2df-8aec-4a4e-9585-52df0fbccf45 }, name: Name(\"external-dns-a702a2df-8aec-4a4e-9585-52df0fbccf45\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 15334e02-b404-488e-9ce7-2a50df738630, kind: Service { id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39 }, name: Name(\"external-dns-7e2aceaf-08e1-46f2-95ec-f9032200ba39\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 88])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 28701ada-e899-402b-a2fd-f7a3fd7c41df, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 0e8e6699-5046-483c-bebf-fffb211be035, kind: Service { id: 28701ada-e899-402b-a2fd-f7a3fd7c41df }, name: Name(\"nexus-28701ada-e899-402b-a2fd-f7a3fd7c41df\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9363e23e-d709-4bd9-bca4-7fbe3ad55235, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 9e4a840d-8987-4208-af8d-b4a2771e9cc4, kind: Service { id: 9363e23e-d709-4bd9-bca4-7fbe3ad55235 }, name: Name(\"nexus-9363e23e-d709-4bd9-bca4-7fbe3ad55235\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 232, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4243c086-1bed-4413-b0c7-7c802e7bc575, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: a0dd5105-fcbd-4055-ae12-c3fb0abcf0fa, kind: Service { id: 4243c086-1bed-4413-b0c7-7c802e7bc575 }, name: Name(\"nexus-4243c086-1bed-4413-b0c7-7c802e7bc575\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 15])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 07eefccc-a16d-41be-9df2-0faa4294ae1c, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a0f87398-93b9-40a8-90b6-7059441c2401, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 2cb47b44-e1c8-4597-82f0-7ee9d5d86893, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 0e3a6bd4-4538-4ccd-a063-9c7342f19203, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f7332e25-0a75-4959-b498-faa862ee2c5f, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: d6729570-1e52-4d84-af84-f4f23623dfb6, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 1d816ece-0442-4979-9d32-7665ef85afde, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 98f9db1c-1595-4f22-a928-321363864abe, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.588956128Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.589577989Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.589596729Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 719635e7-d366-4895-9272-9ade9dc77ba8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.61176916Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 719635e7-d366-4895-9272-9ade9dc77ba8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.611833932Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ae10e87-0030-442b-9981-0b4c70a9c369, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.6315218Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2ae10e87-0030-442b-9981-0b4c70a9c369, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.631576152Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.651138116Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.651206539Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bf26395e-9671-4bee-b96d-1c6f76e85a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.671114264Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bf26395e-9671-4bee-b96d-1c6f76e85a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.671159086Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.690799692Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.690842844Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21544c6f-59af-4a59-a720-d40b64e76f5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.710325016Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 21544c6f-59af-4a59-a720-d40b64e76f5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.710375107Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ddb8e0b6-c193-419e-b893-def90acf2c6e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.729789867Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ddb8e0b6-c193-419e-b893-def90acf2c6e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.729853249Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a702a2df-8aec-4a4e-9585-52df0fbccf45, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.748056328Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a702a2df-8aec-4a4e-9585-52df0fbccf45, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.748112007Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.766737574Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.766795696Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.784710626Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.784783938Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.803955137Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.804014462Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.822141939Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.822195631Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.840662089Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.84072131Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7332e25-0a75-4959-b498-faa862ee2c5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.859092376Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f7332e25-0a75-4959-b498-faa862ee2c5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.859148988Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6729570-1e52-4d84-af84-f4f23623dfb6, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.877857304Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d6729570-1e52-4d84-af84-f4f23623dfb6, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.877915176Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1d816ece-0442-4979-9d32-7665ef85afde, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.89597409Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1d816ece-0442-4979-9d32-7665ef85afde, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.896031682Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 98f9db1c-1595-4f22-a928-321363864abe, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.914726748Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 98f9db1c-1595-4f22-a928-321363864abe, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.91479262Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.935649937Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.935696899Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.95452588Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.954578841Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.974311301Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.974386283Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.97457889Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:00.974759445Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:01.070158058Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:01.070246501Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:01.070267681Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:01.070283722Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:01.070292842Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:01.070301612Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:01.070310812Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:01.070320213Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:01.070338633Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:01.070347284Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:01.070355784Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:01.070364794Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:01.070373425Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:01.070381895Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:01.070386995Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:01.070393995Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:01.070399206Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:01.070419726Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:01.070429936Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:01.070492189Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"f0755912-ac33-4f7c-8ae6-ddf862043871","remote_addr":"[fd00:1122:3344:101::1]:35849","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23\", TimeNotSynchronized), (\"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235\", TimeNotSynchronized), (\"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c\", TimeNotSynchronized), (\"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2\", TimeNotSynchronized), (\"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45\", TimeNotSynchronized), (\"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575\", TimeNotSynchronized), (\"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401\", TimeNotSynchronized), (\"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39\", TimeNotSynchronized), (\"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17\", TimeNotSynchronized), (\"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7\", TimeNotSynchronized), (\"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302\", TimeNotSynchronized), (\"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893\", TimeNotSynchronized), (\"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f\", TimeNotSynchronized), (\"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6\", TimeNotSynchronized), (\"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde\", TimeNotSynchronized), (\"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55\", TimeNotSynchronized), (\"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203\", TimeNotSynchronized), (\"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe\", TimeNotSynchronized), (\"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df\", TimeNotSynchronized)]","latency_us":482408,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:01.070726956Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"216.498366ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f0755912-ac33-4f7c-8ae6-ddf862043871\", \"content-length\": \"124\", \"date\": \"Thu, 15 Feb 2024 05:57:00 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f0755912-ac33-4f7c-8ae6-ddf862043871\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 719635e7-d366-4895-9272-9ade9dc77ba8, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2ae10e87-0030-442b-9981-0b4c70a9c369, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a62bfe46-059b-4ea1-b2a5-7f98d258138d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 44034536-a7b3-4ef5-9ea6-8cd61921d1ee, kind: Service { id: a62bfe46-059b-4ea1-b2a5-7f98d258138d }, name: Name(\"ntp-a62bfe46-059b-4ea1-b2a5-7f98d258138d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 159])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bf26395e-9671-4bee-b96d-1c6f76e85a02, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 21544c6f-59af-4a59-a720-d40b64e76f5d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: ddb8e0b6-c193-419e-b893-def90acf2c6e, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: a702a2df-8aec-4a4e-9585-52df0fbccf45, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 80b71d9a-f960-4a96-9d83-f9c1c3b2663c, kind: Service { id: a702a2df-8aec-4a4e-9585-52df0fbccf45 }, name: Name(\"external-dns-a702a2df-8aec-4a4e-9585-52df0fbccf45\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 15334e02-b404-488e-9ce7-2a50df738630, kind: Service { id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39 }, name: Name(\"external-dns-7e2aceaf-08e1-46f2-95ec-f9032200ba39\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 88])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 28701ada-e899-402b-a2fd-f7a3fd7c41df, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 0e8e6699-5046-483c-bebf-fffb211be035, kind: Service { id: 28701ada-e899-402b-a2fd-f7a3fd7c41df }, name: Name(\"nexus-28701ada-e899-402b-a2fd-f7a3fd7c41df\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9363e23e-d709-4bd9-bca4-7fbe3ad55235, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 9e4a840d-8987-4208-af8d-b4a2771e9cc4, kind: Service { id: 9363e23e-d709-4bd9-bca4-7fbe3ad55235 }, name: Name(\"nexus-9363e23e-d709-4bd9-bca4-7fbe3ad55235\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 232, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4243c086-1bed-4413-b0c7-7c802e7bc575, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: a0dd5105-fcbd-4055-ae12-c3fb0abcf0fa, kind: Service { id: 4243c086-1bed-4413-b0c7-7c802e7bc575 }, name: Name(\"nexus-4243c086-1bed-4413-b0c7-7c802e7bc575\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 15])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 07eefccc-a16d-41be-9df2-0faa4294ae1c, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a0f87398-93b9-40a8-90b6-7059441c2401, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 2cb47b44-e1c8-4597-82f0-7ee9d5d86893, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 0e3a6bd4-4538-4ccd-a063-9c7342f19203, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f7332e25-0a75-4959-b498-faa862ee2c5f, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: d6729570-1e52-4d84-af84-f4f23623dfb6, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 1d816ece-0442-4979-9d32-7665ef85afde, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 98f9db1c-1595-4f22-a928-321363864abe, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:01.28853494Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:01.289264474Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:01.289282795Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 719635e7-d366-4895-9272-9ade9dc77ba8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:01.310541173Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 719635e7-d366-4895-9272-9ade9dc77ba8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:01.310587477Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ae10e87-0030-442b-9981-0b4c70a9c369, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:01.330103059Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2ae10e87-0030-442b-9981-0b4c70a9c369, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:01.330163171Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:01.350552363Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:01.350602575Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bf26395e-9671-4bee-b96d-1c6f76e85a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:01.370768859Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bf26395e-9671-4bee-b96d-1c6f76e85a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:01.37081256Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:01.389823496Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:01.389893369Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21544c6f-59af-4a59-a720-d40b64e76f5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:01.408611755Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 21544c6f-59af-4a59-a720-d40b64e76f5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:01.408674087Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ddb8e0b6-c193-419e-b893-def90acf2c6e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:01.427130025Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ddb8e0b6-c193-419e-b893-def90acf2c6e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:01.427197498Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a702a2df-8aec-4a4e-9585-52df0fbccf45, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:01.445156689Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a702a2df-8aec-4a4e-9585-52df0fbccf45, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:01.445201341Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:01.463374339Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:01.46342104Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:01.481472355Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:01.481521317Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:01.501837196Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:01.501885158Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:01.52049136Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:01.520536452Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:01.538211714Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:01.538269626Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7332e25-0a75-4959-b498-faa862ee2c5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:01.556396173Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f7332e25-0a75-4959-b498-faa862ee2c5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:01.556443664Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6729570-1e52-4d84-af84-f4f23623dfb6, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:01.577142126Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d6729570-1e52-4d84-af84-f4f23623dfb6, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:01.577185198Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1d816ece-0442-4979-9d32-7665ef85afde, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:01.596197914Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1d816ece-0442-4979-9d32-7665ef85afde, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:01.596242815Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 98f9db1c-1595-4f22-a928-321363864abe, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:01.615279752Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 98f9db1c-1595-4f22-a928-321363864abe, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:01.615329254Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:01.633914886Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:01.633960688Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:01.652874301Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:01.652922902Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:01.670921205Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:01.670987397Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:01.671191954Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:01.671321298Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:01.765632095Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:01.765695537Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:01.765704917Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:01.765718757Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:01.765727048Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:01.765733388Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:01.765740618Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:01.765749029Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:01.765757299Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:01.765763619Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:01.765769789Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:01.765777639Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:01.76578373Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:01.7657974Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:01.765822491Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:01.765829031Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:01.765836381Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:01.765843592Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:01.765850972Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:01.765946035Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"038cbb33-868f-4832-969d-5a6317818a72","remote_addr":"[fd00:1122:3344:101::1]:35849","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe\", TimeNotSynchronized), (\"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39\", TimeNotSynchronized), (\"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde\", TimeNotSynchronized), (\"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17\", TimeNotSynchronized), (\"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c\", TimeNotSynchronized), (\"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401\", TimeNotSynchronized), (\"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df\", TimeNotSynchronized), (\"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893\", TimeNotSynchronized), (\"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2\", TimeNotSynchronized), (\"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203\", TimeNotSynchronized), (\"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45\", TimeNotSynchronized), (\"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6\", TimeNotSynchronized), (\"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302\", TimeNotSynchronized), (\"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55\", TimeNotSynchronized), (\"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7\", TimeNotSynchronized), (\"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23\", TimeNotSynchronized), (\"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235\", TimeNotSynchronized), (\"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f\", TimeNotSynchronized), (\"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575\", TimeNotSynchronized)]","latency_us":478196,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:01.766184513Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"131.247172ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"038cbb33-868f-4832-969d-5a6317818a72\", \"content-length\": \"124\", \"date\": \"Thu, 15 Feb 2024 05:57:01 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"038cbb33-868f-4832-969d-5a6317818a72\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 719635e7-d366-4895-9272-9ade9dc77ba8, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2ae10e87-0030-442b-9981-0b4c70a9c369, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a62bfe46-059b-4ea1-b2a5-7f98d258138d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 44034536-a7b3-4ef5-9ea6-8cd61921d1ee, kind: Service { id: a62bfe46-059b-4ea1-b2a5-7f98d258138d }, name: Name(\"ntp-a62bfe46-059b-4ea1-b2a5-7f98d258138d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 159])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bf26395e-9671-4bee-b96d-1c6f76e85a02, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 21544c6f-59af-4a59-a720-d40b64e76f5d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: ddb8e0b6-c193-419e-b893-def90acf2c6e, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: a702a2df-8aec-4a4e-9585-52df0fbccf45, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 80b71d9a-f960-4a96-9d83-f9c1c3b2663c, kind: Service { id: a702a2df-8aec-4a4e-9585-52df0fbccf45 }, name: Name(\"external-dns-a702a2df-8aec-4a4e-9585-52df0fbccf45\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 15334e02-b404-488e-9ce7-2a50df738630, kind: Service { id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39 }, name: Name(\"external-dns-7e2aceaf-08e1-46f2-95ec-f9032200ba39\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 88])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 28701ada-e899-402b-a2fd-f7a3fd7c41df, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 0e8e6699-5046-483c-bebf-fffb211be035, kind: Service { id: 28701ada-e899-402b-a2fd-f7a3fd7c41df }, name: Name(\"nexus-28701ada-e899-402b-a2fd-f7a3fd7c41df\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9363e23e-d709-4bd9-bca4-7fbe3ad55235, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 9e4a840d-8987-4208-af8d-b4a2771e9cc4, kind: Service { id: 9363e23e-d709-4bd9-bca4-7fbe3ad55235 }, name: Name(\"nexus-9363e23e-d709-4bd9-bca4-7fbe3ad55235\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 232, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4243c086-1bed-4413-b0c7-7c802e7bc575, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: a0dd5105-fcbd-4055-ae12-c3fb0abcf0fa, kind: Service { id: 4243c086-1bed-4413-b0c7-7c802e7bc575 }, name: Name(\"nexus-4243c086-1bed-4413-b0c7-7c802e7bc575\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 15])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 07eefccc-a16d-41be-9df2-0faa4294ae1c, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a0f87398-93b9-40a8-90b6-7059441c2401, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 2cb47b44-e1c8-4597-82f0-7ee9d5d86893, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 0e3a6bd4-4538-4ccd-a063-9c7342f19203, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f7332e25-0a75-4959-b498-faa862ee2c5f, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: d6729570-1e52-4d84-af84-f4f23623dfb6, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 1d816ece-0442-4979-9d32-7665ef85afde, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 98f9db1c-1595-4f22-a928-321363864abe, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:01.897998785Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:01.898579944Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:01.898595864Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 719635e7-d366-4895-9272-9ade9dc77ba8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:01.918935474Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 719635e7-d366-4895-9272-9ade9dc77ba8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:01.918987806Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ae10e87-0030-442b-9981-0b4c70a9c369, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:01.938452697Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2ae10e87-0030-442b-9981-0b4c70a9c369, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:01.938509659Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:01.958834008Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:01.95888725Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bf26395e-9671-4bee-b96d-1c6f76e85a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:01.978224527Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bf26395e-9671-4bee-b96d-1c6f76e85a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:01.978288649Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:01.997184251Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:01.997249494Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21544c6f-59af-4a59-a720-d40b64e76f5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:02.016740796Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 21544c6f-59af-4a59-a720-d40b64e76f5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:02.016793837Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ddb8e0b6-c193-419e-b893-def90acf2c6e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:02.034507481Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ddb8e0b6-c193-419e-b893-def90acf2c6e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:02.034557442Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a702a2df-8aec-4a4e-9585-52df0fbccf45, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:02.053179556Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a702a2df-8aec-4a4e-9585-52df0fbccf45, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:02.053234457Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:02.072518962Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:02.072571764Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:02.093237145Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:02.093290517Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:02.11311508Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:02.113176452Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:02.131722743Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:02.131775254Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:02.15077665Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:02.150826352Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7332e25-0a75-4959-b498-faa862ee2c5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:02.169458166Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f7332e25-0a75-4959-b498-faa862ee2c5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:02.169510407Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6729570-1e52-4d84-af84-f4f23623dfb6, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:02.188146821Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d6729570-1e52-4d84-af84-f4f23623dfb6, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:02.188199593Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1d816ece-0442-4979-9d32-7665ef85afde, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:02.206599019Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1d816ece-0442-4979-9d32-7665ef85afde, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:02.206669951Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 98f9db1c-1595-4f22-a928-321363864abe, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:02.225535133Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 98f9db1c-1595-4f22-a928-321363864abe, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:02.225599635Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:02.245338645Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:02.245412377Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:02.264838787Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:02.264911149Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:02.28346449Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:02.283530403Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:02.283722879Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:02.283854313Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:02.3775602Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:02.37765196Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:02.377663641Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:02.377672482Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:02.377681276Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:02.377690134Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:02.377707075Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:02.377718195Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:02.377736596Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:02.377742206Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:02.377749386Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:02.377770867Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:02.377778667Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:02.377785557Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:02.377794348Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:02.377801748Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:02.377807348Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:02.377814128Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:02.377821529Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:02.377843089Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"7098e82c-520f-4d55-aa94-37da37301824","remote_addr":"[fd00:1122:3344:101::1]:35849","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c\", TimeNotSynchronized), (\"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45\", TimeNotSynchronized), (\"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893\", TimeNotSynchronized), (\"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7\", TimeNotSynchronized), (\"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6\", TimeNotSynchronized), (\"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302\", TimeNotSynchronized), (\"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde\", TimeNotSynchronized), (\"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe\", TimeNotSynchronized), (\"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235\", TimeNotSynchronized), (\"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39\", TimeNotSynchronized), (\"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17\", TimeNotSynchronized), (\"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f\", TimeNotSynchronized), (\"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df\", TimeNotSynchronized), (\"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575\", TimeNotSynchronized), (\"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203\", TimeNotSynchronized), (\"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401\", TimeNotSynchronized), (\"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2\", TimeNotSynchronized), (\"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23\", TimeNotSynchronized), (\"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55\", TimeNotSynchronized)]","latency_us":480744,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:02.378083277Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"367.202428ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7098e82c-520f-4d55-aa94-37da37301824\", \"content-length\": \"124\", \"date\": \"Thu, 15 Feb 2024 05:57:01 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"7098e82c-520f-4d55-aa94-37da37301824\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 719635e7-d366-4895-9272-9ade9dc77ba8, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2ae10e87-0030-442b-9981-0b4c70a9c369, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a62bfe46-059b-4ea1-b2a5-7f98d258138d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 44034536-a7b3-4ef5-9ea6-8cd61921d1ee, kind: Service { id: a62bfe46-059b-4ea1-b2a5-7f98d258138d }, name: Name(\"ntp-a62bfe46-059b-4ea1-b2a5-7f98d258138d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 159])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bf26395e-9671-4bee-b96d-1c6f76e85a02, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 21544c6f-59af-4a59-a720-d40b64e76f5d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: ddb8e0b6-c193-419e-b893-def90acf2c6e, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: a702a2df-8aec-4a4e-9585-52df0fbccf45, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 80b71d9a-f960-4a96-9d83-f9c1c3b2663c, kind: Service { id: a702a2df-8aec-4a4e-9585-52df0fbccf45 }, name: Name(\"external-dns-a702a2df-8aec-4a4e-9585-52df0fbccf45\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 15334e02-b404-488e-9ce7-2a50df738630, kind: Service { id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39 }, name: Name(\"external-dns-7e2aceaf-08e1-46f2-95ec-f9032200ba39\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 88])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 28701ada-e899-402b-a2fd-f7a3fd7c41df, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 0e8e6699-5046-483c-bebf-fffb211be035, kind: Service { id: 28701ada-e899-402b-a2fd-f7a3fd7c41df }, name: Name(\"nexus-28701ada-e899-402b-a2fd-f7a3fd7c41df\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9363e23e-d709-4bd9-bca4-7fbe3ad55235, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 9e4a840d-8987-4208-af8d-b4a2771e9cc4, kind: Service { id: 9363e23e-d709-4bd9-bca4-7fbe3ad55235 }, name: Name(\"nexus-9363e23e-d709-4bd9-bca4-7fbe3ad55235\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 232, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4243c086-1bed-4413-b0c7-7c802e7bc575, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: a0dd5105-fcbd-4055-ae12-c3fb0abcf0fa, kind: Service { id: 4243c086-1bed-4413-b0c7-7c802e7bc575 }, name: Name(\"nexus-4243c086-1bed-4413-b0c7-7c802e7bc575\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 15])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 07eefccc-a16d-41be-9df2-0faa4294ae1c, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a0f87398-93b9-40a8-90b6-7059441c2401, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 2cb47b44-e1c8-4597-82f0-7ee9d5d86893, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 0e3a6bd4-4538-4ccd-a063-9c7342f19203, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f7332e25-0a75-4959-b498-faa862ee2c5f, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: d6729570-1e52-4d84-af84-f4f23623dfb6, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 1d816ece-0442-4979-9d32-7665ef85afde, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 98f9db1c-1595-4f22-a928-321363864abe, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:02.74582615Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:02.746412499Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:02.746425019Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 719635e7-d366-4895-9272-9ade9dc77ba8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:02.767754652Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 719635e7-d366-4895-9272-9ade9dc77ba8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:02.767810754Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ae10e87-0030-442b-9981-0b4c70a9c369, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:02.78834804Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2ae10e87-0030-442b-9981-0b4c70a9c369, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:02.788410882Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:02.807543922Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:02.807593874Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bf26395e-9671-4bee-b96d-1c6f76e85a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:02.827000063Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bf26395e-9671-4bee-b96d-1c6f76e85a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:02.827060705Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:02.84512143Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:02.845187662Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21544c6f-59af-4a59-a720-d40b64e76f5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:02.865289424Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 21544c6f-59af-4a59-a720-d40b64e76f5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:02.865345865Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ddb8e0b6-c193-419e-b893-def90acf2c6e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:02.883790754Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ddb8e0b6-c193-419e-b893-def90acf2c6e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:02.883852546Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a702a2df-8aec-4a4e-9585-52df0fbccf45, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:02.902231127Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a702a2df-8aec-4a4e-9585-52df0fbccf45, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:02.902312724Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:02.921555067Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:02.92162205Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:02.939785608Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:02.93983829Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:02.958187384Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:02.958251026Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:02.977274073Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:02.977336285Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:02.995154332Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:02.995225694Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7332e25-0a75-4959-b498-faa862ee2c5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:03.013397323Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f7332e25-0a75-4959-b498-faa862ee2c5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:03.013446604Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6729570-1e52-4d84-af84-f4f23623dfb6, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:03.032331936Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d6729570-1e52-4d84-af84-f4f23623dfb6, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:03.032398649Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1d816ece-0442-4979-9d32-7665ef85afde, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:03.051305991Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1d816ece-0442-4979-9d32-7665ef85afde, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:03.051361793Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 98f9db1c-1595-4f22-a928-321363864abe, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:03.070597547Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 98f9db1c-1595-4f22-a928-321363864abe, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:03.070663759Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:03.088980612Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:03.089038474Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:03.107267394Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:03.107340617Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:03.125217446Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:03.125306638Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:03.125470104Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:03.125623731Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:03.220329068Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:03.220397221Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:03.220413101Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:03.220420031Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:03.220427931Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:03.220434332Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:03.220440732Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:03.220448542Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:03.220471973Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:03.220478593Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:03.220484433Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:03.220491684Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:03.220497694Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:03.220505124Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:03.220517894Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:03.220526105Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:03.220533585Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:03.220539415Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:03.220545325Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:03.220644379Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"abfe3f3b-8c29-492d-a778-e018784bb8b2","remote_addr":"[fd00:1122:3344:101::1]:35849","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2\", TimeNotSynchronized), (\"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe\", TimeNotSynchronized), (\"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7\", TimeNotSynchronized), (\"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6\", TimeNotSynchronized), (\"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39\", TimeNotSynchronized), (\"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f\", TimeNotSynchronized), (\"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893\", TimeNotSynchronized), (\"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17\", TimeNotSynchronized), (\"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55\", TimeNotSynchronized), (\"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203\", TimeNotSynchronized), (\"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235\", TimeNotSynchronized), (\"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575\", TimeNotSynchronized), (\"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302\", TimeNotSynchronized), (\"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401\", TimeNotSynchronized), (\"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde\", TimeNotSynchronized), (\"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23\", TimeNotSynchronized), (\"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45\", TimeNotSynchronized), (\"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df\", TimeNotSynchronized), (\"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c\", TimeNotSynchronized)]","latency_us":475700,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:03.220897547Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"274.886812ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"abfe3f3b-8c29-492d-a778-e018784bb8b2\", \"content-length\": \"124\", \"date\": \"Thu, 15 Feb 2024 05:57:03 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"abfe3f3b-8c29-492d-a778-e018784bb8b2\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 719635e7-d366-4895-9272-9ade9dc77ba8, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2ae10e87-0030-442b-9981-0b4c70a9c369, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a62bfe46-059b-4ea1-b2a5-7f98d258138d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 44034536-a7b3-4ef5-9ea6-8cd61921d1ee, kind: Service { id: a62bfe46-059b-4ea1-b2a5-7f98d258138d }, name: Name(\"ntp-a62bfe46-059b-4ea1-b2a5-7f98d258138d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 159])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bf26395e-9671-4bee-b96d-1c6f76e85a02, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 21544c6f-59af-4a59-a720-d40b64e76f5d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: ddb8e0b6-c193-419e-b893-def90acf2c6e, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: a702a2df-8aec-4a4e-9585-52df0fbccf45, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 80b71d9a-f960-4a96-9d83-f9c1c3b2663c, kind: Service { id: a702a2df-8aec-4a4e-9585-52df0fbccf45 }, name: Name(\"external-dns-a702a2df-8aec-4a4e-9585-52df0fbccf45\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 15334e02-b404-488e-9ce7-2a50df738630, kind: Service { id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39 }, name: Name(\"external-dns-7e2aceaf-08e1-46f2-95ec-f9032200ba39\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 88])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 28701ada-e899-402b-a2fd-f7a3fd7c41df, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 0e8e6699-5046-483c-bebf-fffb211be035, kind: Service { id: 28701ada-e899-402b-a2fd-f7a3fd7c41df }, name: Name(\"nexus-28701ada-e899-402b-a2fd-f7a3fd7c41df\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9363e23e-d709-4bd9-bca4-7fbe3ad55235, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 9e4a840d-8987-4208-af8d-b4a2771e9cc4, kind: Service { id: 9363e23e-d709-4bd9-bca4-7fbe3ad55235 }, name: Name(\"nexus-9363e23e-d709-4bd9-bca4-7fbe3ad55235\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 232, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4243c086-1bed-4413-b0c7-7c802e7bc575, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: a0dd5105-fcbd-4055-ae12-c3fb0abcf0fa, kind: Service { id: 4243c086-1bed-4413-b0c7-7c802e7bc575 }, name: Name(\"nexus-4243c086-1bed-4413-b0c7-7c802e7bc575\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 15])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 07eefccc-a16d-41be-9df2-0faa4294ae1c, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a0f87398-93b9-40a8-90b6-7059441c2401, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 2cb47b44-e1c8-4597-82f0-7ee9d5d86893, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 0e3a6bd4-4538-4ccd-a063-9c7342f19203, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f7332e25-0a75-4959-b498-faa862ee2c5f, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: d6729570-1e52-4d84-af84-f4f23623dfb6, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 1d816ece-0442-4979-9d32-7665ef85afde, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 98f9db1c-1595-4f22-a928-321363864abe, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:03.49635865Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:03.497485247Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:03.497512798Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 719635e7-d366-4895-9272-9ade9dc77ba8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:03.520324049Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 719635e7-d366-4895-9272-9ade9dc77ba8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:03.520384221Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ae10e87-0030-442b-9981-0b4c70a9c369, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:03.541199687Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2ae10e87-0030-442b-9981-0b4c70a9c369, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:03.541255009Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:03.560803302Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:03.560854304Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bf26395e-9671-4bee-b96d-1c6f76e85a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:03.580862683Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bf26395e-9671-4bee-b96d-1c6f76e85a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:03.580914625Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:03.600624094Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:03.600675136Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21544c6f-59af-4a59-a720-d40b64e76f5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:03.619496456Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 21544c6f-59af-4a59-a720-d40b64e76f5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:03.619545728Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ddb8e0b6-c193-419e-b893-def90acf2c6e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:03.638054237Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ddb8e0b6-c193-419e-b893-def90acf2c6e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:03.638127379Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a702a2df-8aec-4a4e-9585-52df0fbccf45, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:03.656593858Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a702a2df-8aec-4a4e-9585-52df0fbccf45, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:03.656647879Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:03.675367676Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:03.675426508Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:03.69431119Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:03.694370742Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:03.714032002Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:03.714095392Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:03.73378048Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:03.733837842Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:03.753351545Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:03.753418057Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7332e25-0a75-4959-b498-faa862ee2c5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:03.77233784Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f7332e25-0a75-4959-b498-faa862ee2c5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:03.772390372Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6729570-1e52-4d84-af84-f4f23623dfb6, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:03.791185911Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d6729570-1e52-4d84-af84-f4f23623dfb6, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:03.791241813Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1d816ece-0442-4979-9d32-7665ef85afde, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:03.810309581Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1d816ece-0442-4979-9d32-7665ef85afde, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:03.810380873Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 98f9db1c-1595-4f22-a928-321363864abe, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:03.828953235Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 98f9db1c-1595-4f22-a928-321363864abe, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:03.829007116Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:03.847760944Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:03.850161923Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:03.867035739Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:03.867095021Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:03.885937212Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:03.886017444Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:03.886222981Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:03.886378466Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:03.977232969Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:03.977293441Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:03.977307667Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:03.977315101Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:03.977322706Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:03.977345993Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:03.977352403Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:03.977360383Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:03.977366813Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:03.977373893Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:03.977380034Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:03.977387654Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:03.977396184Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:03.977404014Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:03.977411595Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:03.977417655Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:03.977421835Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:03.977427765Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:03.977431925Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:03.977505178Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"5b9fad1f-c47c-4a80-88c5-5b3dbb83b9fb","remote_addr":"[fd00:1122:3344:101::1]:35849","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45\", TimeNotSynchronized), (\"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401\", TimeNotSynchronized), (\"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55\", TimeNotSynchronized), (\"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203\", TimeNotSynchronized), (\"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575\", TimeNotSynchronized), (\"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302\", TimeNotSynchronized), (\"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe\", TimeNotSynchronized), (\"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df\", TimeNotSynchronized), (\"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235\", TimeNotSynchronized), (\"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17\", TimeNotSynchronized), (\"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f\", TimeNotSynchronized), (\"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39\", TimeNotSynchronized), (\"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde\", TimeNotSynchronized), (\"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7\", TimeNotSynchronized), (\"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23\", TimeNotSynchronized), (\"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2\", TimeNotSynchronized), (\"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893\", TimeNotSynchronized), (\"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c\", TimeNotSynchronized), (\"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6\", TimeNotSynchronized)]","latency_us":481988,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:03.977745246Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"196.329858ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5b9fad1f-c47c-4a80-88c5-5b3dbb83b9fb\", \"content-length\": \"124\", \"date\": \"Thu, 15 Feb 2024 05:57:03 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"5b9fad1f-c47c-4a80-88c5-5b3dbb83b9fb\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 719635e7-d366-4895-9272-9ade9dc77ba8, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2ae10e87-0030-442b-9981-0b4c70a9c369, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a62bfe46-059b-4ea1-b2a5-7f98d258138d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 44034536-a7b3-4ef5-9ea6-8cd61921d1ee, kind: Service { id: a62bfe46-059b-4ea1-b2a5-7f98d258138d }, name: Name(\"ntp-a62bfe46-059b-4ea1-b2a5-7f98d258138d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 159])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bf26395e-9671-4bee-b96d-1c6f76e85a02, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 21544c6f-59af-4a59-a720-d40b64e76f5d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: ddb8e0b6-c193-419e-b893-def90acf2c6e, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: a702a2df-8aec-4a4e-9585-52df0fbccf45, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 80b71d9a-f960-4a96-9d83-f9c1c3b2663c, kind: Service { id: a702a2df-8aec-4a4e-9585-52df0fbccf45 }, name: Name(\"external-dns-a702a2df-8aec-4a4e-9585-52df0fbccf45\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 15334e02-b404-488e-9ce7-2a50df738630, kind: Service { id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39 }, name: Name(\"external-dns-7e2aceaf-08e1-46f2-95ec-f9032200ba39\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 88])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 28701ada-e899-402b-a2fd-f7a3fd7c41df, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 0e8e6699-5046-483c-bebf-fffb211be035, kind: Service { id: 28701ada-e899-402b-a2fd-f7a3fd7c41df }, name: Name(\"nexus-28701ada-e899-402b-a2fd-f7a3fd7c41df\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9363e23e-d709-4bd9-bca4-7fbe3ad55235, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 9e4a840d-8987-4208-af8d-b4a2771e9cc4, kind: Service { id: 9363e23e-d709-4bd9-bca4-7fbe3ad55235 }, name: Name(\"nexus-9363e23e-d709-4bd9-bca4-7fbe3ad55235\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 232, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4243c086-1bed-4413-b0c7-7c802e7bc575, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: a0dd5105-fcbd-4055-ae12-c3fb0abcf0fa, kind: Service { id: 4243c086-1bed-4413-b0c7-7c802e7bc575 }, name: Name(\"nexus-4243c086-1bed-4413-b0c7-7c802e7bc575\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 15])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 07eefccc-a16d-41be-9df2-0faa4294ae1c, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a0f87398-93b9-40a8-90b6-7059441c2401, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 2cb47b44-e1c8-4597-82f0-7ee9d5d86893, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 0e3a6bd4-4538-4ccd-a063-9c7342f19203, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f7332e25-0a75-4959-b498-faa862ee2c5f, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: d6729570-1e52-4d84-af84-f4f23623dfb6, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 1d816ece-0442-4979-9d32-7665ef85afde, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 98f9db1c-1595-4f22-a928-321363864abe, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:04.174405452Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:04.175080774Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:04.175108125Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 719635e7-d366-4895-9272-9ade9dc77ba8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:04.196917564Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 719635e7-d366-4895-9272-9ade9dc77ba8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:04.196975766Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ae10e87-0030-442b-9981-0b4c70a9c369, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:04.216510989Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2ae10e87-0030-442b-9981-0b4c70a9c369, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:04.216560811Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:04.236256979Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:04.236321391Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bf26395e-9671-4bee-b96d-1c6f76e85a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:04.254591623Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bf26395e-9671-4bee-b96d-1c6f76e85a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:04.254644575Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:04.273414693Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:04.273464945Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21544c6f-59af-4a59-a720-d40b64e76f5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:04.292675918Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 21544c6f-59af-4a59-a720-d40b64e76f5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:04.29273412Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ddb8e0b6-c193-419e-b893-def90acf2c6e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:04.31219741Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ddb8e0b6-c193-419e-b893-def90acf2c6e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:04.312249792Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a702a2df-8aec-4a4e-9585-52df0fbccf45, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:04.331079552Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a702a2df-8aec-4a4e-9585-52df0fbccf45, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:04.331129564Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:04.351208226Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:04.351262877Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:04.370434049Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:04.370487361Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:04.389768276Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:04.389818557Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:04.408646057Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:04.411006355Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:04.427190418Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:04.42725021Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7332e25-0a75-4959-b498-faa862ee2c5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:04.446561766Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f7332e25-0a75-4959-b498-faa862ee2c5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:04.446626168Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6729570-1e52-4d84-af84-f4f23623dfb6, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:04.465400237Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d6729570-1e52-4d84-af84-f4f23623dfb6, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:04.465470409Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1d816ece-0442-4979-9d32-7665ef85afde, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:04.484202076Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1d816ece-0442-4979-9d32-7665ef85afde, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:04.484255898Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 98f9db1c-1595-4f22-a928-321363864abe, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:04.502646463Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 98f9db1c-1595-4f22-a928-321363864abe, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:04.502710145Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:04.522117605Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:04.522174537Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:04.541154592Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:04.541210004Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:04.559845058Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:04.55992205Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:04.560089886Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:04.56022337Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:04.655197088Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:04.655275891Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:04.655301162Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:04.655312092Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:04.655319972Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:04.655327622Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:04.655333933Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:04.655341533Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:04.655347633Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:04.655355623Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:04.655364153Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:04.655377534Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:04.655383454Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:04.655389334Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:04.655393785Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:04.655402575Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:04.655407055Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:04.655419486Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:04.655427236Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:04.655492098Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"12e020be-3206-4384-aa14-cb69e757e603","remote_addr":"[fd00:1122:3344:101::1]:35849","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893\", TimeNotSynchronized), (\"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f\", TimeNotSynchronized), (\"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203\", TimeNotSynchronized), (\"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45\", TimeNotSynchronized), (\"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39\", TimeNotSynchronized), (\"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7\", TimeNotSynchronized), (\"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575\", TimeNotSynchronized), (\"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde\", TimeNotSynchronized), (\"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23\", TimeNotSynchronized), (\"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302\", TimeNotSynchronized), (\"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235\", TimeNotSynchronized), (\"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401\", TimeNotSynchronized), (\"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55\", TimeNotSynchronized), (\"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c\", TimeNotSynchronized), (\"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2\", TimeNotSynchronized), (\"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df\", TimeNotSynchronized), (\"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6\", TimeNotSynchronized), (\"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe\", TimeNotSynchronized), (\"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17\", TimeNotSynchronized)]","latency_us":481912,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:04.655745206Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"416.50604ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"12e020be-3206-4384-aa14-cb69e757e603\", \"content-length\": \"124\", \"date\": \"Thu, 15 Feb 2024 05:57:04 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"12e020be-3206-4384-aa14-cb69e757e603\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 719635e7-d366-4895-9272-9ade9dc77ba8, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2ae10e87-0030-442b-9981-0b4c70a9c369, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a62bfe46-059b-4ea1-b2a5-7f98d258138d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 44034536-a7b3-4ef5-9ea6-8cd61921d1ee, kind: Service { id: a62bfe46-059b-4ea1-b2a5-7f98d258138d }, name: Name(\"ntp-a62bfe46-059b-4ea1-b2a5-7f98d258138d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 159])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bf26395e-9671-4bee-b96d-1c6f76e85a02, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 21544c6f-59af-4a59-a720-d40b64e76f5d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: ddb8e0b6-c193-419e-b893-def90acf2c6e, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: a702a2df-8aec-4a4e-9585-52df0fbccf45, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 80b71d9a-f960-4a96-9d83-f9c1c3b2663c, kind: Service { id: a702a2df-8aec-4a4e-9585-52df0fbccf45 }, name: Name(\"external-dns-a702a2df-8aec-4a4e-9585-52df0fbccf45\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 15334e02-b404-488e-9ce7-2a50df738630, kind: Service { id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39 }, name: Name(\"external-dns-7e2aceaf-08e1-46f2-95ec-f9032200ba39\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 88])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 28701ada-e899-402b-a2fd-f7a3fd7c41df, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 0e8e6699-5046-483c-bebf-fffb211be035, kind: Service { id: 28701ada-e899-402b-a2fd-f7a3fd7c41df }, name: Name(\"nexus-28701ada-e899-402b-a2fd-f7a3fd7c41df\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9363e23e-d709-4bd9-bca4-7fbe3ad55235, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 9e4a840d-8987-4208-af8d-b4a2771e9cc4, kind: Service { id: 9363e23e-d709-4bd9-bca4-7fbe3ad55235 }, name: Name(\"nexus-9363e23e-d709-4bd9-bca4-7fbe3ad55235\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 232, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4243c086-1bed-4413-b0c7-7c802e7bc575, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: a0dd5105-fcbd-4055-ae12-c3fb0abcf0fa, kind: Service { id: 4243c086-1bed-4413-b0c7-7c802e7bc575 }, name: Name(\"nexus-4243c086-1bed-4413-b0c7-7c802e7bc575\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 15])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 07eefccc-a16d-41be-9df2-0faa4294ae1c, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a0f87398-93b9-40a8-90b6-7059441c2401, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 2cb47b44-e1c8-4597-82f0-7ee9d5d86893, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 0e3a6bd4-4538-4ccd-a063-9c7342f19203, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f7332e25-0a75-4959-b498-faa862ee2c5f, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: d6729570-1e52-4d84-af84-f4f23623dfb6, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 1d816ece-0442-4979-9d32-7665ef85afde, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 98f9db1c-1595-4f22-a928-321363864abe, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:05.071913033Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:05.072537444Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:05.072556525Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 719635e7-d366-4895-9272-9ade9dc77ba8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:05.09488578Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 719635e7-d366-4895-9272-9ade9dc77ba8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:05.094961512Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ae10e87-0030-442b-9981-0b4c70a9c369, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:05.115281142Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2ae10e87-0030-442b-9981-0b4c70a9c369, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:05.115343634Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:05.133672567Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:05.1337401Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bf26395e-9671-4bee-b96d-1c6f76e85a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:05.151528336Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bf26395e-9671-4bee-b96d-1c6f76e85a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:05.151589838Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:05.169504978Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:05.16956292Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21544c6f-59af-4a59-a720-d40b64e76f5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:05.1896021Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 21544c6f-59af-4a59-a720-d40b64e76f5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:05.189672192Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ddb8e0b6-c193-419e-b893-def90acf2c6e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:05.208362287Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ddb8e0b6-c193-419e-b893-def90acf2c6e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:05.20842549Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a702a2df-8aec-4a4e-9585-52df0fbccf45, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:05.226694121Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a702a2df-8aec-4a4e-9585-52df0fbccf45, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:05.226748763Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:05.245556543Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:05.245620425Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:05.264440965Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:05.264505627Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:05.282940324Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:05.282991656Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:05.301306189Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:05.301361511Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:05.319766057Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:05.322163446Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7332e25-0a75-4959-b498-faa862ee2c5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:05.338277557Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f7332e25-0a75-4959-b498-faa862ee2c5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:05.338340609Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6729570-1e52-4d84-af84-f4f23623dfb6, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:05.356643212Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d6729570-1e52-4d84-af84-f4f23623dfb6, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:05.356711294Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1d816ece-0442-4979-9d32-7665ef85afde, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:05.376551857Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1d816ece-0442-4979-9d32-7665ef85afde, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:05.376607789Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 98f9db1c-1595-4f22-a928-321363864abe, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:05.395084158Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 98f9db1c-1595-4f22-a928-321363864abe, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:05.39515479Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:05.415465429Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:05.415519811Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:05.435441837Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:05.435495369Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:05.453875284Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:05.453949516Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:05.454197465Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:05.45436103Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:05.546818646Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:05.546896958Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:05.546906058Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:05.546917799Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:05.546924269Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:05.546930519Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:05.546935389Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:05.546939979Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:05.54694794Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:05.54695545Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:05.5469618Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:05.54696953Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:05.546977311Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:05.546983411Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:05.546990971Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:05.547005701Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:05.547012022Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:05.547069088Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:05.547077004Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:05.547144696Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"367e9883-dffd-4ade-89b2-a1995ced619f","remote_addr":"[fd00:1122:3344:101::1]:35849","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893\", TimeNotSynchronized), (\"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575\", TimeNotSynchronized), (\"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f\", TimeNotSynchronized), (\"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6\", TimeNotSynchronized), (\"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23\", TimeNotSynchronized), (\"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401\", TimeNotSynchronized), (\"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c\", TimeNotSynchronized), (\"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39\", TimeNotSynchronized), (\"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df\", TimeNotSynchronized), (\"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203\", TimeNotSynchronized), (\"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45\", TimeNotSynchronized), (\"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302\", TimeNotSynchronized), (\"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7\", TimeNotSynchronized), (\"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde\", TimeNotSynchronized), (\"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe\", TimeNotSynchronized), (\"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235\", TimeNotSynchronized), (\"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2\", TimeNotSynchronized), (\"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55\", TimeNotSynchronized), (\"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17\", TimeNotSynchronized)]","latency_us":476099,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:05.547393005Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"685.51831ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"367e9883-dffd-4ade-89b2-a1995ced619f\", \"content-length\": \"124\", \"date\": \"Thu, 15 Feb 2024 05:57:05 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"367e9883-dffd-4ade-89b2-a1995ced619f\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 719635e7-d366-4895-9272-9ade9dc77ba8, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2ae10e87-0030-442b-9981-0b4c70a9c369, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a62bfe46-059b-4ea1-b2a5-7f98d258138d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 44034536-a7b3-4ef5-9ea6-8cd61921d1ee, kind: Service { id: a62bfe46-059b-4ea1-b2a5-7f98d258138d }, name: Name(\"ntp-a62bfe46-059b-4ea1-b2a5-7f98d258138d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 159])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bf26395e-9671-4bee-b96d-1c6f76e85a02, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 21544c6f-59af-4a59-a720-d40b64e76f5d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: ddb8e0b6-c193-419e-b893-def90acf2c6e, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: a702a2df-8aec-4a4e-9585-52df0fbccf45, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 80b71d9a-f960-4a96-9d83-f9c1c3b2663c, kind: Service { id: a702a2df-8aec-4a4e-9585-52df0fbccf45 }, name: Name(\"external-dns-a702a2df-8aec-4a4e-9585-52df0fbccf45\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 15334e02-b404-488e-9ce7-2a50df738630, kind: Service { id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39 }, name: Name(\"external-dns-7e2aceaf-08e1-46f2-95ec-f9032200ba39\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 88])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 28701ada-e899-402b-a2fd-f7a3fd7c41df, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 0e8e6699-5046-483c-bebf-fffb211be035, kind: Service { id: 28701ada-e899-402b-a2fd-f7a3fd7c41df }, name: Name(\"nexus-28701ada-e899-402b-a2fd-f7a3fd7c41df\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9363e23e-d709-4bd9-bca4-7fbe3ad55235, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 9e4a840d-8987-4208-af8d-b4a2771e9cc4, kind: Service { id: 9363e23e-d709-4bd9-bca4-7fbe3ad55235 }, name: Name(\"nexus-9363e23e-d709-4bd9-bca4-7fbe3ad55235\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 232, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4243c086-1bed-4413-b0c7-7c802e7bc575, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: a0dd5105-fcbd-4055-ae12-c3fb0abcf0fa, kind: Service { id: 4243c086-1bed-4413-b0c7-7c802e7bc575 }, name: Name(\"nexus-4243c086-1bed-4413-b0c7-7c802e7bc575\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 15])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 07eefccc-a16d-41be-9df2-0faa4294ae1c, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a0f87398-93b9-40a8-90b6-7059441c2401, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 2cb47b44-e1c8-4597-82f0-7ee9d5d86893, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 0e3a6bd4-4538-4ccd-a063-9c7342f19203, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f7332e25-0a75-4959-b498-faa862ee2c5f, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: d6729570-1e52-4d84-af84-f4f23623dfb6, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 1d816ece-0442-4979-9d32-7665ef85afde, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 98f9db1c-1595-4f22-a928-321363864abe, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:06.233058187Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:06.233680028Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:06.233703389Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 719635e7-d366-4895-9272-9ade9dc77ba8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:06.255721744Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 719635e7-d366-4895-9272-9ade9dc77ba8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:06.255785936Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ae10e87-0030-442b-9981-0b4c70a9c369, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:06.276675774Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2ae10e87-0030-442b-9981-0b4c70a9c369, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:06.276740506Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:06.296789947Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:06.296848858Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bf26395e-9671-4bee-b96d-1c6f76e85a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:06.316060541Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bf26395e-9671-4bee-b96d-1c6f76e85a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:06.316122423Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:06.335592585Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:06.335664717Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21544c6f-59af-4a59-a720-d40b64e76f5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:06.354442635Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 21544c6f-59af-4a59-a720-d40b64e76f5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:06.354501788Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ddb8e0b6-c193-419e-b893-def90acf2c6e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:06.373078969Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ddb8e0b6-c193-419e-b893-def90acf2c6e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:06.373153562Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a702a2df-8aec-4a4e-9585-52df0fbccf45, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:06.391537717Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a702a2df-8aec-4a4e-9585-52df0fbccf45, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:06.391596739Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:06.41134551Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:06.411410942Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:06.43110294Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:06.431160212Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:06.450382626Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:06.450458028Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:06.469877698Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:06.46994126Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:06.488938105Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:06.489000787Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7332e25-0a75-4959-b498-faa862ee2c5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:06.507317051Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f7332e25-0a75-4959-b498-faa862ee2c5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:06.507382073Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6729570-1e52-4d84-af84-f4f23623dfb6, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:06.525719267Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d6729570-1e52-4d84-af84-f4f23623dfb6, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:06.525776989Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1d816ece-0442-4979-9d32-7665ef85afde, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:06.54614246Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1d816ece-0442-4979-9d32-7665ef85afde, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:06.546205862Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 98f9db1c-1595-4f22-a928-321363864abe, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:06.565642322Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 98f9db1c-1595-4f22-a928-321363864abe, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:06.565713834Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:06.584286606Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:06.584344498Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:06.60292119Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:06.602981552Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:06.622577937Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:06.62266465Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:06.622837106Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:06.6229838Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:06.720008466Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:06.720069098Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:06.720077129Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:06.720082859Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:06.720089519Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:06.720095589Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:06.72010211Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:06.72011004Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:06.72011783Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:06.72012565Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:06.72013346Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:06.720139701Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:06.720156571Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:06.720164381Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:06.720172272Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:06.720179252Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:06.720187122Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:06.720192002Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:06.720198102Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:06.720277815Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"11258261-1ab5-47b6-ac5f-480cf9924c0a","remote_addr":"[fd00:1122:3344:101::1]:35849","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe\", TimeNotSynchronized), (\"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45\", TimeNotSynchronized), (\"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f\", TimeNotSynchronized), (\"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23\", TimeNotSynchronized), (\"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55\", TimeNotSynchronized), (\"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893\", TimeNotSynchronized), (\"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17\", TimeNotSynchronized), (\"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39\", TimeNotSynchronized), (\"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c\", TimeNotSynchronized), (\"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575\", TimeNotSynchronized), (\"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2\", TimeNotSynchronized), (\"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6\", TimeNotSynchronized), (\"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7\", TimeNotSynchronized), (\"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde\", TimeNotSynchronized), (\"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235\", TimeNotSynchronized), (\"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df\", TimeNotSynchronized), (\"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302\", TimeNotSynchronized), (\"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401\", TimeNotSynchronized), (\"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203\", TimeNotSynchronized)]","latency_us":488110,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:06.720518243Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"468.115547ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"11258261-1ab5-47b6-ac5f-480cf9924c0a\", \"content-length\": \"124\", \"date\": \"Thu, 15 Feb 2024 05:57:06 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"11258261-1ab5-47b6-ac5f-480cf9924c0a\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 719635e7-d366-4895-9272-9ade9dc77ba8, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2ae10e87-0030-442b-9981-0b4c70a9c369, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a62bfe46-059b-4ea1-b2a5-7f98d258138d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 44034536-a7b3-4ef5-9ea6-8cd61921d1ee, kind: Service { id: a62bfe46-059b-4ea1-b2a5-7f98d258138d }, name: Name(\"ntp-a62bfe46-059b-4ea1-b2a5-7f98d258138d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 159])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bf26395e-9671-4bee-b96d-1c6f76e85a02, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 21544c6f-59af-4a59-a720-d40b64e76f5d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: ddb8e0b6-c193-419e-b893-def90acf2c6e, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: a702a2df-8aec-4a4e-9585-52df0fbccf45, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 80b71d9a-f960-4a96-9d83-f9c1c3b2663c, kind: Service { id: a702a2df-8aec-4a4e-9585-52df0fbccf45 }, name: Name(\"external-dns-a702a2df-8aec-4a4e-9585-52df0fbccf45\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 15334e02-b404-488e-9ce7-2a50df738630, kind: Service { id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39 }, name: Name(\"external-dns-7e2aceaf-08e1-46f2-95ec-f9032200ba39\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 88])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 28701ada-e899-402b-a2fd-f7a3fd7c41df, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 0e8e6699-5046-483c-bebf-fffb211be035, kind: Service { id: 28701ada-e899-402b-a2fd-f7a3fd7c41df }, name: Name(\"nexus-28701ada-e899-402b-a2fd-f7a3fd7c41df\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9363e23e-d709-4bd9-bca4-7fbe3ad55235, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 9e4a840d-8987-4208-af8d-b4a2771e9cc4, kind: Service { id: 9363e23e-d709-4bd9-bca4-7fbe3ad55235 }, name: Name(\"nexus-9363e23e-d709-4bd9-bca4-7fbe3ad55235\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 232, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4243c086-1bed-4413-b0c7-7c802e7bc575, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: a0dd5105-fcbd-4055-ae12-c3fb0abcf0fa, kind: Service { id: 4243c086-1bed-4413-b0c7-7c802e7bc575 }, name: Name(\"nexus-4243c086-1bed-4413-b0c7-7c802e7bc575\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 15])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 07eefccc-a16d-41be-9df2-0faa4294ae1c, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a0f87398-93b9-40a8-90b6-7059441c2401, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 2cb47b44-e1c8-4597-82f0-7ee9d5d86893, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 0e3a6bd4-4538-4ccd-a063-9c7342f19203, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f7332e25-0a75-4959-b498-faa862ee2c5f, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: d6729570-1e52-4d84-af84-f4f23623dfb6, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 1d816ece-0442-4979-9d32-7665ef85afde, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 98f9db1c-1595-4f22-a928-321363864abe, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:07.187717831Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:07.188322121Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:07.188337982Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 719635e7-d366-4895-9272-9ade9dc77ba8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:07.209746227Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 719635e7-d366-4895-9272-9ade9dc77ba8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:07.211734052Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ae10e87-0030-442b-9981-0b4c70a9c369, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:07.230657526Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2ae10e87-0030-442b-9981-0b4c70a9c369, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:07.230715617Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:07.250494549Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:07.250558731Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bf26395e-9671-4bee-b96d-1c6f76e85a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:07.268677008Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bf26395e-9671-4bee-b96d-1c6f76e85a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:07.271032245Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:07.288078467Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:07.288137269Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21544c6f-59af-4a59-a720-d40b64e76f5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:07.306217914Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 21544c6f-59af-4a59-a720-d40b64e76f5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:07.306277892Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ddb8e0b6-c193-419e-b893-def90acf2c6e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:07.324755835Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ddb8e0b6-c193-419e-b893-def90acf2c6e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:07.324818527Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a702a2df-8aec-4a4e-9585-52df0fbccf45, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:07.343235004Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a702a2df-8aec-4a4e-9585-52df0fbccf45, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:07.343300496Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:07.363632346Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:07.363686437Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:07.384094249Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:07.384152241Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:07.403898072Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:07.403963014Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:07.424113588Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:07.42418942Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:07.444293332Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:07.444357444Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7332e25-0a75-4959-b498-faa862ee2c5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:07.464816088Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f7332e25-0a75-4959-b498-faa862ee2c5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:07.46487287Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6729570-1e52-4d84-af84-f4f23623dfb6, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:07.48369825Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d6729570-1e52-4d84-af84-f4f23623dfb6, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:07.483751432Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1d816ece-0442-4979-9d32-7665ef85afde, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:07.503247174Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1d816ece-0442-4979-9d32-7665ef85afde, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:07.503312676Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 98f9db1c-1595-4f22-a928-321363864abe, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:07.521852557Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 98f9db1c-1595-4f22-a928-321363864abe, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:07.521909299Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:07.541201294Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:07.541268456Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:07.562528237Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:07.562594887Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:07.584053936Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:07.584125388Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:07.584296534Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:07.584454119Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:07.682894311Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:07.682969264Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:07.682979294Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:07.682986804Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:07.682993344Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:07.683001455Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:07.683007495Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:07.683013585Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:07.683027006Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:07.683042756Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:07.683050586Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:07.683056327Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:07.683062027Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:07.683075277Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:07.683083377Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:07.683088208Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:07.683092728Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:07.683098438Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:07.683107238Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:07.683189901Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"a836cc40-e662-46d4-b91c-82e879acf634","remote_addr":"[fd00:1122:3344:101::1]:35849","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39\", TimeNotSynchronized), (\"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23\", TimeNotSynchronized), (\"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401\", TimeNotSynchronized), (\"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df\", TimeNotSynchronized), (\"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c\", TimeNotSynchronized), (\"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45\", TimeNotSynchronized), (\"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575\", TimeNotSynchronized), (\"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7\", TimeNotSynchronized), (\"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893\", TimeNotSynchronized), (\"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2\", TimeNotSynchronized), (\"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f\", TimeNotSynchronized), (\"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6\", TimeNotSynchronized), (\"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55\", TimeNotSynchronized), (\"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302\", TimeNotSynchronized), (\"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235\", TimeNotSynchronized), (\"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203\", TimeNotSynchronized), (\"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17\", TimeNotSynchronized), (\"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde\", TimeNotSynchronized), (\"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe\", TimeNotSynchronized)]","latency_us":496396,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:07.683442769Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"726.368599ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a836cc40-e662-46d4-b91c-82e879acf634\", \"content-length\": \"124\", \"date\": \"Thu, 15 Feb 2024 05:57:07 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a836cc40-e662-46d4-b91c-82e879acf634\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 719635e7-d366-4895-9272-9ade9dc77ba8, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2ae10e87-0030-442b-9981-0b4c70a9c369, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a62bfe46-059b-4ea1-b2a5-7f98d258138d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 44034536-a7b3-4ef5-9ea6-8cd61921d1ee, kind: Service { id: a62bfe46-059b-4ea1-b2a5-7f98d258138d }, name: Name(\"ntp-a62bfe46-059b-4ea1-b2a5-7f98d258138d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 159])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bf26395e-9671-4bee-b96d-1c6f76e85a02, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 21544c6f-59af-4a59-a720-d40b64e76f5d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: ddb8e0b6-c193-419e-b893-def90acf2c6e, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: a702a2df-8aec-4a4e-9585-52df0fbccf45, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 80b71d9a-f960-4a96-9d83-f9c1c3b2663c, kind: Service { id: a702a2df-8aec-4a4e-9585-52df0fbccf45 }, name: Name(\"external-dns-a702a2df-8aec-4a4e-9585-52df0fbccf45\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 15334e02-b404-488e-9ce7-2a50df738630, kind: Service { id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39 }, name: Name(\"external-dns-7e2aceaf-08e1-46f2-95ec-f9032200ba39\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 88])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 28701ada-e899-402b-a2fd-f7a3fd7c41df, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 0e8e6699-5046-483c-bebf-fffb211be035, kind: Service { id: 28701ada-e899-402b-a2fd-f7a3fd7c41df }, name: Name(\"nexus-28701ada-e899-402b-a2fd-f7a3fd7c41df\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9363e23e-d709-4bd9-bca4-7fbe3ad55235, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 9e4a840d-8987-4208-af8d-b4a2771e9cc4, kind: Service { id: 9363e23e-d709-4bd9-bca4-7fbe3ad55235 }, name: Name(\"nexus-9363e23e-d709-4bd9-bca4-7fbe3ad55235\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 232, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4243c086-1bed-4413-b0c7-7c802e7bc575, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: a0dd5105-fcbd-4055-ae12-c3fb0abcf0fa, kind: Service { id: 4243c086-1bed-4413-b0c7-7c802e7bc575 }, name: Name(\"nexus-4243c086-1bed-4413-b0c7-7c802e7bc575\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 15])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 07eefccc-a16d-41be-9df2-0faa4294ae1c, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a0f87398-93b9-40a8-90b6-7059441c2401, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 2cb47b44-e1c8-4597-82f0-7ee9d5d86893, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 0e3a6bd4-4538-4ccd-a063-9c7342f19203, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f7332e25-0a75-4959-b498-faa862ee2c5f, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: d6729570-1e52-4d84-af84-f4f23623dfb6, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 1d816ece-0442-4979-9d32-7665ef85afde, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 98f9db1c-1595-4f22-a928-321363864abe, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:08.409020788Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:08.409713521Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:08.409736811Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 719635e7-d366-4895-9272-9ade9dc77ba8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:08.432830242Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 719635e7-d366-4895-9272-9ade9dc77ba8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:08.432892384Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ae10e87-0030-442b-9981-0b4c70a9c369, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:08.454177795Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2ae10e87-0030-442b-9981-0b4c70a9c369, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:08.454240687Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:08.472786868Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:08.47285709Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bf26395e-9671-4bee-b96d-1c6f76e85a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:08.49166747Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bf26395e-9671-4bee-b96d-1c6f76e85a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:08.491726462Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:08.511641988Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:08.51169813Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21544c6f-59af-4a59-a720-d40b64e76f5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:08.530173048Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 21544c6f-59af-4a59-a720-d40b64e76f5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:08.53022728Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ddb8e0b6-c193-419e-b893-def90acf2c6e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:08.548945236Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ddb8e0b6-c193-419e-b893-def90acf2c6e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:08.548998078Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a702a2df-8aec-4a4e-9585-52df0fbccf45, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:08.568318225Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a702a2df-8aec-4a4e-9585-52df0fbccf45, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:08.568381167Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:08.587855818Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:08.58791164Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:08.607122503Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:08.607182095Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:08.626542162Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:08.626616565Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:08.645265559Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:08.645321101Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:08.664345437Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:08.66441569Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7332e25-0a75-4959-b498-faa862ee2c5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:08.685048209Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f7332e25-0a75-4959-b498-faa862ee2c5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:08.685102131Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6729570-1e52-4d84-af84-f4f23623dfb6, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:08.704416217Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d6729570-1e52-4d84-af84-f4f23623dfb6, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:08.70448674Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1d816ece-0442-4979-9d32-7665ef85afde, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:08.723892729Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1d816ece-0442-4979-9d32-7665ef85afde, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:08.723949231Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 98f9db1c-1595-4f22-a928-321363864abe, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:08.743507615Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 98f9db1c-1595-4f22-a928-321363864abe, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:08.743565957Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:08.763023988Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:08.763079329Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:08.78221766Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:08.782271685Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:08.800771271Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:08.800859024Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:08.80104938Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:08.801231255Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:08.899476932Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:08.899541574Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:08.899547804Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:08.899560975Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:08.899568975Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:08.899575005Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:08.899594276Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:08.899600626Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:08.899606786Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:08.899614157Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:08.899620077Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:08.899625857Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:08.899633357Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:08.899647138Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:08.899651988Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:08.899658278Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:08.899662778Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:08.899666868Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:08.899671388Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:08.899753061Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"9edb66df-7cbe-4769-8be6-49edbd5dcac4","remote_addr":"[fd00:1122:3344:101::1]:35849","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17\", TimeNotSynchronized), (\"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2\", TimeNotSynchronized), (\"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7\", TimeNotSynchronized), (\"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893\", TimeNotSynchronized), (\"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575\", TimeNotSynchronized), (\"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203\", TimeNotSynchronized), (\"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39\", TimeNotSynchronized), (\"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f\", TimeNotSynchronized), (\"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde\", TimeNotSynchronized), (\"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe\", TimeNotSynchronized), (\"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23\", TimeNotSynchronized), (\"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55\", TimeNotSynchronized), (\"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c\", TimeNotSynchronized), (\"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6\", TimeNotSynchronized), (\"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235\", TimeNotSynchronized), (\"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401\", TimeNotSynchronized), (\"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df\", TimeNotSynchronized), (\"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302\", TimeNotSynchronized), (\"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45\", TimeNotSynchronized)]","latency_us":491569,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:08.90001654Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"612.217159ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9edb66df-7cbe-4769-8be6-49edbd5dcac4\", \"content-length\": \"124\", \"date\": \"Thu, 15 Feb 2024 05:57:08 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"9edb66df-7cbe-4769-8be6-49edbd5dcac4\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 719635e7-d366-4895-9272-9ade9dc77ba8, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2ae10e87-0030-442b-9981-0b4c70a9c369, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a62bfe46-059b-4ea1-b2a5-7f98d258138d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 44034536-a7b3-4ef5-9ea6-8cd61921d1ee, kind: Service { id: a62bfe46-059b-4ea1-b2a5-7f98d258138d }, name: Name(\"ntp-a62bfe46-059b-4ea1-b2a5-7f98d258138d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 159])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bf26395e-9671-4bee-b96d-1c6f76e85a02, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 21544c6f-59af-4a59-a720-d40b64e76f5d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: ddb8e0b6-c193-419e-b893-def90acf2c6e, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: a702a2df-8aec-4a4e-9585-52df0fbccf45, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 80b71d9a-f960-4a96-9d83-f9c1c3b2663c, kind: Service { id: a702a2df-8aec-4a4e-9585-52df0fbccf45 }, name: Name(\"external-dns-a702a2df-8aec-4a4e-9585-52df0fbccf45\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 15334e02-b404-488e-9ce7-2a50df738630, kind: Service { id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39 }, name: Name(\"external-dns-7e2aceaf-08e1-46f2-95ec-f9032200ba39\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 88])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 28701ada-e899-402b-a2fd-f7a3fd7c41df, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 0e8e6699-5046-483c-bebf-fffb211be035, kind: Service { id: 28701ada-e899-402b-a2fd-f7a3fd7c41df }, name: Name(\"nexus-28701ada-e899-402b-a2fd-f7a3fd7c41df\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9363e23e-d709-4bd9-bca4-7fbe3ad55235, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 9e4a840d-8987-4208-af8d-b4a2771e9cc4, kind: Service { id: 9363e23e-d709-4bd9-bca4-7fbe3ad55235 }, name: Name(\"nexus-9363e23e-d709-4bd9-bca4-7fbe3ad55235\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 232, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4243c086-1bed-4413-b0c7-7c802e7bc575, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: a0dd5105-fcbd-4055-ae12-c3fb0abcf0fa, kind: Service { id: 4243c086-1bed-4413-b0c7-7c802e7bc575 }, name: Name(\"nexus-4243c086-1bed-4413-b0c7-7c802e7bc575\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 15])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 07eefccc-a16d-41be-9df2-0faa4294ae1c, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a0f87398-93b9-40a8-90b6-7059441c2401, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 2cb47b44-e1c8-4597-82f0-7ee9d5d86893, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 0e3a6bd4-4538-4ccd-a063-9c7342f19203, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f7332e25-0a75-4959-b498-faa862ee2c5f, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: d6729570-1e52-4d84-af84-f4f23623dfb6, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 1d816ece-0442-4979-9d32-7665ef85afde, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 98f9db1c-1595-4f22-a928-321363864abe, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:09.511872092Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:09.512478392Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:09.512499603Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 719635e7-d366-4895-9272-9ade9dc77ba8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:09.534615152Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 719635e7-d366-4895-9272-9ade9dc77ba8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:09.534679794Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ae10e87-0030-442b-9981-0b4c70a9c369, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:09.554421414Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2ae10e87-0030-442b-9981-0b4c70a9c369, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:09.554485296Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:09.572762448Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:09.5728314Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bf26395e-9671-4bee-b96d-1c6f76e85a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:09.591385121Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bf26395e-9671-4bee-b96d-1c6f76e85a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:09.591443773Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:09.610332166Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:09.610387827Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21544c6f-59af-4a59-a720-d40b64e76f5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:09.628617808Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 21544c6f-59af-4a59-a720-d40b64e76f5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:09.628676299Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ddb8e0b6-c193-419e-b893-def90acf2c6e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:09.649964051Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ddb8e0b6-c193-419e-b893-def90acf2c6e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:09.650030803Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a702a2df-8aec-4a4e-9585-52df0fbccf45, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:09.668732959Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a702a2df-8aec-4a4e-9585-52df0fbccf45, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:09.668791161Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:09.687678803Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:09.687733075Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:09.706863165Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:09.706920507Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:09.725270741Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:09.725325493Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:09.745156616Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:09.747542185Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:09.763765029Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:09.763822451Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7332e25-0a75-4959-b498-faa862ee2c5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:09.782449275Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f7332e25-0a75-4959-b498-faa862ee2c5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:09.784878615Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6729570-1e52-4d84-af84-f4f23623dfb6, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:09.801058377Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d6729570-1e52-4d84-af84-f4f23623dfb6, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:09.801114249Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1d816ece-0442-4979-9d32-7665ef85afde, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:09.820528719Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1d816ece-0442-4979-9d32-7665ef85afde, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:09.82058364Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 98f9db1c-1595-4f22-a928-321363864abe, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:09.839161723Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 98f9db1c-1595-4f22-a928-321363864abe, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:09.839217614Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:09.857413674Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:09.857471275Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:09.875727527Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:09.875786499Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:09.894253825Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:09.894322689Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:09.894515786Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:09.89465817Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:09.991122768Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:09.99118566Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:09.99119357Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:09.99121032Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:09.991217231Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:09.991224761Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:09.991237712Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:09.991244212Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:09.991251802Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:09.991259412Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:09.991266173Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:09.991278483Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:09.991286103Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:09.991293793Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:09.991299874Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:09.991307434Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:09.991313654Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:09.991318104Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:09.991332525Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:09.991407647Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"df7f26dd-a7e9-4478-9a5d-8ae3682f51ca","remote_addr":"[fd00:1122:3344:101::1]:35849","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6\", TimeNotSynchronized), (\"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde\", TimeNotSynchronized), (\"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23\", TimeNotSynchronized), (\"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893\", TimeNotSynchronized), (\"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235\", TimeNotSynchronized), (\"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39\", TimeNotSynchronized), (\"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203\", TimeNotSynchronized), (\"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7\", TimeNotSynchronized), (\"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55\", TimeNotSynchronized), (\"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575\", TimeNotSynchronized), (\"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe\", TimeNotSynchronized), (\"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17\", TimeNotSynchronized), (\"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45\", TimeNotSynchronized), (\"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c\", TimeNotSynchronized), (\"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f\", TimeNotSynchronized), (\"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df\", TimeNotSynchronized), (\"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302\", TimeNotSynchronized), (\"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2\", TimeNotSynchronized), (\"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401\", TimeNotSynchronized)]","latency_us":480427,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:09.991648825Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"911.163863ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"df7f26dd-a7e9-4478-9a5d-8ae3682f51ca\", \"content-length\": \"124\", \"date\": \"Thu, 15 Feb 2024 05:57:09 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"df7f26dd-a7e9-4478-9a5d-8ae3682f51ca\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 719635e7-d366-4895-9272-9ade9dc77ba8, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2ae10e87-0030-442b-9981-0b4c70a9c369, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a62bfe46-059b-4ea1-b2a5-7f98d258138d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 44034536-a7b3-4ef5-9ea6-8cd61921d1ee, kind: Service { id: a62bfe46-059b-4ea1-b2a5-7f98d258138d }, name: Name(\"ntp-a62bfe46-059b-4ea1-b2a5-7f98d258138d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 159])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bf26395e-9671-4bee-b96d-1c6f76e85a02, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 21544c6f-59af-4a59-a720-d40b64e76f5d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: ddb8e0b6-c193-419e-b893-def90acf2c6e, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: a702a2df-8aec-4a4e-9585-52df0fbccf45, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 80b71d9a-f960-4a96-9d83-f9c1c3b2663c, kind: Service { id: a702a2df-8aec-4a4e-9585-52df0fbccf45 }, name: Name(\"external-dns-a702a2df-8aec-4a4e-9585-52df0fbccf45\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 15334e02-b404-488e-9ce7-2a50df738630, kind: Service { id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39 }, name: Name(\"external-dns-7e2aceaf-08e1-46f2-95ec-f9032200ba39\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 88])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 28701ada-e899-402b-a2fd-f7a3fd7c41df, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 0e8e6699-5046-483c-bebf-fffb211be035, kind: Service { id: 28701ada-e899-402b-a2fd-f7a3fd7c41df }, name: Name(\"nexus-28701ada-e899-402b-a2fd-f7a3fd7c41df\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9363e23e-d709-4bd9-bca4-7fbe3ad55235, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 9e4a840d-8987-4208-af8d-b4a2771e9cc4, kind: Service { id: 9363e23e-d709-4bd9-bca4-7fbe3ad55235 }, name: Name(\"nexus-9363e23e-d709-4bd9-bca4-7fbe3ad55235\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 232, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4243c086-1bed-4413-b0c7-7c802e7bc575, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: a0dd5105-fcbd-4055-ae12-c3fb0abcf0fa, kind: Service { id: 4243c086-1bed-4413-b0c7-7c802e7bc575 }, name: Name(\"nexus-4243c086-1bed-4413-b0c7-7c802e7bc575\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 15])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 07eefccc-a16d-41be-9df2-0faa4294ae1c, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a0f87398-93b9-40a8-90b6-7059441c2401, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 2cb47b44-e1c8-4597-82f0-7ee9d5d86893, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 0e3a6bd4-4538-4ccd-a063-9c7342f19203, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f7332e25-0a75-4959-b498-faa862ee2c5f, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: d6729570-1e52-4d84-af84-f4f23623dfb6, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 1d816ece-0442-4979-9d32-7665ef85afde, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 98f9db1c-1595-4f22-a928-321363864abe, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:10.900757938Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:10.901378499Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:10.90139844Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 719635e7-d366-4895-9272-9ade9dc77ba8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:10.922718122Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 719635e7-d366-4895-9272-9ade9dc77ba8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:10.925109641Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ae10e87-0030-442b-9981-0b4c70a9c369, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:10.944083196Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2ae10e87-0030-442b-9981-0b4c70a9c369, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:10.944142738Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:10.963558357Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:10.963620109Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bf26395e-9671-4bee-b96d-1c6f76e85a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:10.98217293Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bf26395e-9671-4bee-b96d-1c6f76e85a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:10.982241522Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:11.002190739Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:11.002249732Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21544c6f-59af-4a59-a720-d40b64e76f5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:11.021188923Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 21544c6f-59af-4a59-a720-d40b64e76f5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:11.021271497Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ddb8e0b6-c193-419e-b893-def90acf2c6e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:11.039496137Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ddb8e0b6-c193-419e-b893-def90acf2c6e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:11.039555489Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a702a2df-8aec-4a4e-9585-52df0fbccf45, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:11.060251751Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a702a2df-8aec-4a4e-9585-52df0fbccf45, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:11.060309483Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:11.079905778Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:11.07996779Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:11.098924765Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:11.098980465Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:11.118051575Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:11.118121237Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:11.138119066Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:11.138181318Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:11.157636418Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:11.15769396Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7332e25-0a75-4959-b498-faa862ee2c5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:11.177063888Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f7332e25-0a75-4959-b498-faa862ee2c5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:11.17712575Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6729570-1e52-4d84-af84-f4f23623dfb6, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:11.196113886Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d6729570-1e52-4d84-af84-f4f23623dfb6, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:11.196177078Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1d816ece-0442-4979-9d32-7665ef85afde, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:11.216321581Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1d816ece-0442-4979-9d32-7665ef85afde, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:11.216371313Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 98f9db1c-1595-4f22-a928-321363864abe, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:11.235078919Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 98f9db1c-1595-4f22-a928-321363864abe, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:11.235140921Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:11.255003836Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:11.255069628Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:11.276510424Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:11.276570206Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:11.296030137Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:11.296102849Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:11.296297096Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:11.296443472Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:11.391410589Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:11.391486251Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:11.391496721Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:11.391504601Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:11.391512402Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:11.391518192Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:11.391525682Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:11.391531752Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:11.391539373Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:11.391556373Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:11.391562963Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:11.391570874Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:11.391576964Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:11.391584364Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:11.391591334Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:11.391611035Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:11.391617555Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:11.391623485Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:11.391629446Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:11.391688898Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"a7f3021c-19ea-4a31-8ca0-19f08cb21a41","remote_addr":"[fd00:1122:3344:101::1]:35849","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55\", TimeNotSynchronized), (\"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235\", TimeNotSynchronized), (\"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c\", TimeNotSynchronized), (\"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6\", TimeNotSynchronized), (\"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2\", TimeNotSynchronized), (\"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39\", TimeNotSynchronized), (\"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe\", TimeNotSynchronized), (\"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203\", TimeNotSynchronized), (\"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302\", TimeNotSynchronized), (\"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17\", TimeNotSynchronized), (\"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde\", TimeNotSynchronized), (\"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45\", TimeNotSynchronized), (\"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df\", TimeNotSynchronized), (\"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575\", TimeNotSynchronized), (\"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401\", TimeNotSynchronized), (\"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893\", TimeNotSynchronized), (\"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f\", TimeNotSynchronized), (\"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23\", TimeNotSynchronized), (\"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7\", TimeNotSynchronized)]","latency_us":491846,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:11.391987177Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"1.694976518s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a7f3021c-19ea-4a31-8ca0-19f08cb21a41\", \"content-length\": \"124\", \"date\": \"Thu, 15 Feb 2024 05:57:11 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a7f3021c-19ea-4a31-8ca0-19f08cb21a41\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 719635e7-d366-4895-9272-9ade9dc77ba8, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2ae10e87-0030-442b-9981-0b4c70a9c369, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a62bfe46-059b-4ea1-b2a5-7f98d258138d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 44034536-a7b3-4ef5-9ea6-8cd61921d1ee, kind: Service { id: a62bfe46-059b-4ea1-b2a5-7f98d258138d }, name: Name(\"ntp-a62bfe46-059b-4ea1-b2a5-7f98d258138d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 159])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bf26395e-9671-4bee-b96d-1c6f76e85a02, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 21544c6f-59af-4a59-a720-d40b64e76f5d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: ddb8e0b6-c193-419e-b893-def90acf2c6e, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: a702a2df-8aec-4a4e-9585-52df0fbccf45, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 80b71d9a-f960-4a96-9d83-f9c1c3b2663c, kind: Service { id: a702a2df-8aec-4a4e-9585-52df0fbccf45 }, name: Name(\"external-dns-a702a2df-8aec-4a4e-9585-52df0fbccf45\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 15334e02-b404-488e-9ce7-2a50df738630, kind: Service { id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39 }, name: Name(\"external-dns-7e2aceaf-08e1-46f2-95ec-f9032200ba39\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 88])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 28701ada-e899-402b-a2fd-f7a3fd7c41df, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 0e8e6699-5046-483c-bebf-fffb211be035, kind: Service { id: 28701ada-e899-402b-a2fd-f7a3fd7c41df }, name: Name(\"nexus-28701ada-e899-402b-a2fd-f7a3fd7c41df\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9363e23e-d709-4bd9-bca4-7fbe3ad55235, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 9e4a840d-8987-4208-af8d-b4a2771e9cc4, kind: Service { id: 9363e23e-d709-4bd9-bca4-7fbe3ad55235 }, name: Name(\"nexus-9363e23e-d709-4bd9-bca4-7fbe3ad55235\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 232, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4243c086-1bed-4413-b0c7-7c802e7bc575, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: a0dd5105-fcbd-4055-ae12-c3fb0abcf0fa, kind: Service { id: 4243c086-1bed-4413-b0c7-7c802e7bc575 }, name: Name(\"nexus-4243c086-1bed-4413-b0c7-7c802e7bc575\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 15])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 07eefccc-a16d-41be-9df2-0faa4294ae1c, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a0f87398-93b9-40a8-90b6-7059441c2401, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 2cb47b44-e1c8-4597-82f0-7ee9d5d86893, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 0e3a6bd4-4538-4ccd-a063-9c7342f19203, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f7332e25-0a75-4959-b498-faa862ee2c5f, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: d6729570-1e52-4d84-af84-f4f23623dfb6, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 1d816ece-0442-4979-9d32-7665ef85afde, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 98f9db1c-1595-4f22-a928-321363864abe, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:13.082839668Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:13.083409287Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:13.083419418Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 719635e7-d366-4895-9272-9ade9dc77ba8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:13.104928576Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 719635e7-d366-4895-9272-9ade9dc77ba8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:13.107317565Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ae10e87-0030-442b-9981-0b4c70a9c369, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:13.124375527Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2ae10e87-0030-442b-9981-0b4c70a9c369, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:13.124440889Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:13.145366418Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:13.147727586Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bf26395e-9671-4bee-b96d-1c6f76e85a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:13.165970237Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bf26395e-9671-4bee-b96d-1c6f76e85a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:13.166035749Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:13.185130948Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:13.187573078Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21544c6f-59af-4a59-a720-d40b64e76f5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:13.205592231Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 21544c6f-59af-4a59-a720-d40b64e76f5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:13.208027262Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ddb8e0b6-c193-419e-b893-def90acf2c6e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:13.225972423Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ddb8e0b6-c193-419e-b893-def90acf2c6e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:13.226036845Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a702a2df-8aec-4a4e-9585-52df0fbccf45, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:13.247059047Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a702a2df-8aec-4a4e-9585-52df0fbccf45, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:13.247118878Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:13.267725198Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:13.270118987Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:13.286716943Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:13.286777136Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:13.30908664Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:13.309147902Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:13.328388586Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:13.330758304Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:13.350192714Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:13.350252206Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7332e25-0a75-4959-b498-faa862ee2c5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:13.369324864Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f7332e25-0a75-4959-b498-faa862ee2c5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:13.371662392Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6729570-1e52-4d84-af84-f4f23623dfb6, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:13.38953947Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d6729570-1e52-4d84-af84-f4f23623dfb6, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:13.389606792Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1d816ece-0442-4979-9d32-7665ef85afde, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:13.410260773Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1d816ece-0442-4979-9d32-7665ef85afde, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:13.410318395Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 98f9db1c-1595-4f22-a928-321363864abe, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:13.429697243Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 98f9db1c-1595-4f22-a928-321363864abe, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:13.429753085Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:13.449798185Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:13.449855947Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:13.469292817Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:13.46937156Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:13.489369788Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:13.489455325Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:13.489629357Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:13.489768091Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:13.587240522Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:13.587306954Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:13.587317044Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:13.587323375Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:13.587328625Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:13.587333455Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:13.587341885Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:13.587348766Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:13.587360026Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:13.587368106Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:13.587376297Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:13.587384367Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:13.587408447Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:13.587416048Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:13.587424148Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:13.587431508Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:13.587437508Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:13.587442139Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:13.587448279Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:13.587529751Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"a621a7cd-c3e0-4377-aeb9-12a0519dac66","remote_addr":"[fd00:1122:3344:101::1]:35849","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45\", TimeNotSynchronized), (\"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17\", TimeNotSynchronized), (\"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c\", TimeNotSynchronized), (\"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2\", TimeNotSynchronized), (\"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6\", TimeNotSynchronized), (\"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302\", TimeNotSynchronized), (\"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235\", TimeNotSynchronized), (\"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39\", TimeNotSynchronized), (\"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575\", TimeNotSynchronized), (\"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401\", TimeNotSynchronized), (\"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893\", TimeNotSynchronized), (\"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203\", TimeNotSynchronized), (\"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f\", TimeNotSynchronized), (\"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde\", TimeNotSynchronized), (\"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe\", TimeNotSynchronized), (\"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55\", TimeNotSynchronized), (\"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df\", TimeNotSynchronized), (\"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7\", TimeNotSynchronized), (\"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23\", TimeNotSynchronized)]","latency_us":505673,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:13.5877794Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"1.192297989s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a621a7cd-c3e0-4377-aeb9-12a0519dac66\", \"content-length\": \"124\", \"date\": \"Thu, 15 Feb 2024 05:57:13 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a621a7cd-c3e0-4377-aeb9-12a0519dac66\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 719635e7-d366-4895-9272-9ade9dc77ba8, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2ae10e87-0030-442b-9981-0b4c70a9c369, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a62bfe46-059b-4ea1-b2a5-7f98d258138d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 44034536-a7b3-4ef5-9ea6-8cd61921d1ee, kind: Service { id: a62bfe46-059b-4ea1-b2a5-7f98d258138d }, name: Name(\"ntp-a62bfe46-059b-4ea1-b2a5-7f98d258138d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 159])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bf26395e-9671-4bee-b96d-1c6f76e85a02, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 21544c6f-59af-4a59-a720-d40b64e76f5d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: ddb8e0b6-c193-419e-b893-def90acf2c6e, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: a702a2df-8aec-4a4e-9585-52df0fbccf45, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 80b71d9a-f960-4a96-9d83-f9c1c3b2663c, kind: Service { id: a702a2df-8aec-4a4e-9585-52df0fbccf45 }, name: Name(\"external-dns-a702a2df-8aec-4a4e-9585-52df0fbccf45\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 15334e02-b404-488e-9ce7-2a50df738630, kind: Service { id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39 }, name: Name(\"external-dns-7e2aceaf-08e1-46f2-95ec-f9032200ba39\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 88])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 28701ada-e899-402b-a2fd-f7a3fd7c41df, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 0e8e6699-5046-483c-bebf-fffb211be035, kind: Service { id: 28701ada-e899-402b-a2fd-f7a3fd7c41df }, name: Name(\"nexus-28701ada-e899-402b-a2fd-f7a3fd7c41df\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9363e23e-d709-4bd9-bca4-7fbe3ad55235, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 9e4a840d-8987-4208-af8d-b4a2771e9cc4, kind: Service { id: 9363e23e-d709-4bd9-bca4-7fbe3ad55235 }, name: Name(\"nexus-9363e23e-d709-4bd9-bca4-7fbe3ad55235\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 232, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4243c086-1bed-4413-b0c7-7c802e7bc575, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: a0dd5105-fcbd-4055-ae12-c3fb0abcf0fa, kind: Service { id: 4243c086-1bed-4413-b0c7-7c802e7bc575 }, name: Name(\"nexus-4243c086-1bed-4413-b0c7-7c802e7bc575\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 15])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 07eefccc-a16d-41be-9df2-0faa4294ae1c, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a0f87398-93b9-40a8-90b6-7059441c2401, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 2cb47b44-e1c8-4597-82f0-7ee9d5d86893, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 0e3a6bd4-4538-4ccd-a063-9c7342f19203, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f7332e25-0a75-4959-b498-faa862ee2c5f, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: d6729570-1e52-4d84-af84-f4f23623dfb6, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 1d816ece-0442-4979-9d32-7665ef85afde, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 98f9db1c-1595-4f22-a928-321363864abe, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:14.778205119Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:14.778890211Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:14.778912582Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 719635e7-d366-4895-9272-9ade9dc77ba8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:14.801634621Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 719635e7-d366-4895-9272-9ade9dc77ba8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:14.801707213Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ae10e87-0030-442b-9981-0b4c70a9c369, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:14.820382888Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2ae10e87-0030-442b-9981-0b4c70a9c369, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:14.82044534Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:14.841154052Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:14.841226735Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bf26395e-9671-4bee-b96d-1c6f76e85a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:14.86172063Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bf26395e-9671-4bee-b96d-1c6f76e85a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:14.861777902Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:14.882456593Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:14.882518645Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21544c6f-59af-4a59-a720-d40b64e76f5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:14.902287286Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 21544c6f-59af-4a59-a720-d40b64e76f5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:14.902344678Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ddb8e0b6-c193-419e-b893-def90acf2c6e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:14.920534777Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ddb8e0b6-c193-419e-b893-def90acf2c6e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:14.920590289Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a702a2df-8aec-4a4e-9585-52df0fbccf45, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:14.938757327Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a702a2df-8aec-4a4e-9585-52df0fbccf45, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:14.941161096Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:14.959292513Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:14.959356836Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:14.978077032Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:14.978138224Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:14.998243287Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:15.000638555Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:15.018973439Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:15.019043481Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:15.041251218Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:15.041309465Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7332e25-0a75-4959-b498-faa862ee2c5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:15.060647732Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f7332e25-0a75-4959-b498-faa862ee2c5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:15.060711654Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6729570-1e52-4d84-af84-f4f23623dfb6, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:15.080290679Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d6729570-1e52-4d84-af84-f4f23623dfb6, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:15.080358901Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1d816ece-0442-4979-9d32-7665ef85afde, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:15.101203007Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1d816ece-0442-4979-9d32-7665ef85afde, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:15.101263809Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 98f9db1c-1595-4f22-a928-321363864abe, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:15.11917966Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 98f9db1c-1595-4f22-a928-321363864abe, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:15.119228761Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:15.139725256Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:15.139784768Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:15.158851046Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:15.158915838Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:15.180009953Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:15.180089216Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:15.180280292Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:15.180420937Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:15.276808542Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:15.276864293Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:15.276873694Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:15.276886274Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:15.276891394Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:15.276897714Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:15.276904025Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:15.276922335Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:15.276930255Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:15.276956696Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:15.276964637Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:15.276972457Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:15.276980577Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:15.276988357Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:15.276994358Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:15.277002268Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:15.277006438Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:15.277012988Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:15.277017418Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:15.277094111Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"8a96ff69-1bf3-4848-abf2-dbcabf1e3284","remote_addr":"[fd00:1122:3344:101::1]:35849","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55\", TimeNotSynchronized), (\"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df\", TimeNotSynchronized), (\"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7\", TimeNotSynchronized), (\"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39\", TimeNotSynchronized), (\"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575\", TimeNotSynchronized), (\"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c\", TimeNotSynchronized), (\"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2\", TimeNotSynchronized), (\"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23\", TimeNotSynchronized), (\"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203\", TimeNotSynchronized), (\"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401\", TimeNotSynchronized), (\"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17\", TimeNotSynchronized), (\"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe\", TimeNotSynchronized), (\"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde\", TimeNotSynchronized), (\"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6\", TimeNotSynchronized), (\"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302\", TimeNotSynchronized), (\"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893\", TimeNotSynchronized), (\"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45\", TimeNotSynchronized), (\"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f\", TimeNotSynchronized), (\"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235\", TimeNotSynchronized)]","latency_us":499769,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:15.277329209Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"2.245444835s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8a96ff69-1bf3-4848-abf2-dbcabf1e3284\", \"content-length\": \"124\", \"date\": \"Thu, 15 Feb 2024 05:57:14 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"8a96ff69-1bf3-4848-abf2-dbcabf1e3284\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 719635e7-d366-4895-9272-9ade9dc77ba8, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2ae10e87-0030-442b-9981-0b4c70a9c369, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a62bfe46-059b-4ea1-b2a5-7f98d258138d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 44034536-a7b3-4ef5-9ea6-8cd61921d1ee, kind: Service { id: a62bfe46-059b-4ea1-b2a5-7f98d258138d }, name: Name(\"ntp-a62bfe46-059b-4ea1-b2a5-7f98d258138d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 159])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bf26395e-9671-4bee-b96d-1c6f76e85a02, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 21544c6f-59af-4a59-a720-d40b64e76f5d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: ddb8e0b6-c193-419e-b893-def90acf2c6e, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: a702a2df-8aec-4a4e-9585-52df0fbccf45, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 80b71d9a-f960-4a96-9d83-f9c1c3b2663c, kind: Service { id: a702a2df-8aec-4a4e-9585-52df0fbccf45 }, name: Name(\"external-dns-a702a2df-8aec-4a4e-9585-52df0fbccf45\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 15334e02-b404-488e-9ce7-2a50df738630, kind: Service { id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39 }, name: Name(\"external-dns-7e2aceaf-08e1-46f2-95ec-f9032200ba39\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 88])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 28701ada-e899-402b-a2fd-f7a3fd7c41df, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 0e8e6699-5046-483c-bebf-fffb211be035, kind: Service { id: 28701ada-e899-402b-a2fd-f7a3fd7c41df }, name: Name(\"nexus-28701ada-e899-402b-a2fd-f7a3fd7c41df\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9363e23e-d709-4bd9-bca4-7fbe3ad55235, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 9e4a840d-8987-4208-af8d-b4a2771e9cc4, kind: Service { id: 9363e23e-d709-4bd9-bca4-7fbe3ad55235 }, name: Name(\"nexus-9363e23e-d709-4bd9-bca4-7fbe3ad55235\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 232, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4243c086-1bed-4413-b0c7-7c802e7bc575, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: a0dd5105-fcbd-4055-ae12-c3fb0abcf0fa, kind: Service { id: 4243c086-1bed-4413-b0c7-7c802e7bc575 }, name: Name(\"nexus-4243c086-1bed-4413-b0c7-7c802e7bc575\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 15])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 07eefccc-a16d-41be-9df2-0faa4294ae1c, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a0f87398-93b9-40a8-90b6-7059441c2401, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 2cb47b44-e1c8-4597-82f0-7ee9d5d86893, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 0e3a6bd4-4538-4ccd-a063-9c7342f19203, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f7332e25-0a75-4959-b498-faa862ee2c5f, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: d6729570-1e52-4d84-af84-f4f23623dfb6, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 1d816ece-0442-4979-9d32-7665ef85afde, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 98f9db1c-1595-4f22-a928-321363864abe, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:17.51798547Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:17.518645922Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:17.518673532Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 719635e7-d366-4895-9272-9ade9dc77ba8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:17.541190514Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 719635e7-d366-4895-9272-9ade9dc77ba8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:17.541260116Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ae10e87-0030-442b-9981-0b4c70a9c369, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:17.562078952Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2ae10e87-0030-442b-9981-0b4c70a9c369, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:17.562145954Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:17.581382208Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:17.583731005Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bf26395e-9671-4bee-b96d-1c6f76e85a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:17.601429028Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bf26395e-9671-4bee-b96d-1c6f76e85a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:17.60148339Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:17.624118105Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:17.624189248Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21544c6f-59af-4a59-a720-d40b64e76f5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:17.643197372Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 21544c6f-59af-4a59-a720-d40b64e76f5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:17.643250846Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ddb8e0b6-c193-419e-b893-def90acf2c6e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:17.662232981Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ddb8e0b6-c193-419e-b893-def90acf2c6e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:17.662286093Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a702a2df-8aec-4a4e-9585-52df0fbccf45, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:17.68103131Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a702a2df-8aec-4a4e-9585-52df0fbccf45, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:17.681094727Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:17.701417631Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:17.701480214Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:17.722912079Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:17.722973696Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:17.743850179Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:17.743930922Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:17.763970092Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:17.764026314Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:17.783758964Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:17.783813323Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7332e25-0a75-4959-b498-faa862ee2c5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:17.803321458Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f7332e25-0a75-4959-b498-faa862ee2c5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:17.80337793Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6729570-1e52-4d84-af84-f4f23623dfb6, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:17.822901483Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d6729570-1e52-4d84-af84-f4f23623dfb6, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:17.825363694Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1d816ece-0442-4979-9d32-7665ef85afde, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:17.842308142Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1d816ece-0442-4979-9d32-7665ef85afde, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:17.842367154Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 98f9db1c-1595-4f22-a928-321363864abe, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:17.862041772Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 98f9db1c-1595-4f22-a928-321363864abe, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:17.862107394Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:17.882707413Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:17.882761744Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:17.902830665Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:17.902886227Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:17.921960666Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:17.922032338Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:17.922226314Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:17.922363809Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:18.019019602Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:18.019095845Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:18.019104795Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:18.019111105Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:18.019119066Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:18.019131286Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:18.019156606Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:18.019170134Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:18.019177296Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:18.019181469Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:18.019191618Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:18.019196378Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:18.019206789Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:18.019213119Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:18.019218639Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:18.019224569Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:18.019231629Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:18.01924751Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:18.0192522Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:18.019297932Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"03772dc4-946c-4f66-955b-b1ac56f02b69","remote_addr":"[fd00:1122:3344:101::1]:35849","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f\", TimeNotSynchronized), (\"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7\", TimeNotSynchronized), (\"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302\", TimeNotSynchronized), (\"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575\", TimeNotSynchronized), (\"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401\", TimeNotSynchronized), (\"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45\", TimeNotSynchronized), (\"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893\", TimeNotSynchronized), (\"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203\", TimeNotSynchronized), (\"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6\", TimeNotSynchronized), (\"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde\", TimeNotSynchronized), (\"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe\", TimeNotSynchronized), (\"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55\", TimeNotSynchronized), (\"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39\", TimeNotSynchronized), (\"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c\", TimeNotSynchronized), (\"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235\", TimeNotSynchronized), (\"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17\", TimeNotSynchronized), (\"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23\", TimeNotSynchronized), (\"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2\", TimeNotSynchronized), (\"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df\", TimeNotSynchronized)]","latency_us":502202,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:18.019535169Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"3.048638311s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"03772dc4-946c-4f66-955b-b1ac56f02b69\", \"content-length\": \"124\", \"date\": \"Thu, 15 Feb 2024 05:57:17 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"03772dc4-946c-4f66-955b-b1ac56f02b69\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 719635e7-d366-4895-9272-9ade9dc77ba8, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2ae10e87-0030-442b-9981-0b4c70a9c369, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a62bfe46-059b-4ea1-b2a5-7f98d258138d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 44034536-a7b3-4ef5-9ea6-8cd61921d1ee, kind: Service { id: a62bfe46-059b-4ea1-b2a5-7f98d258138d }, name: Name(\"ntp-a62bfe46-059b-4ea1-b2a5-7f98d258138d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 159])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bf26395e-9671-4bee-b96d-1c6f76e85a02, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 21544c6f-59af-4a59-a720-d40b64e76f5d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: ddb8e0b6-c193-419e-b893-def90acf2c6e, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: a702a2df-8aec-4a4e-9585-52df0fbccf45, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 80b71d9a-f960-4a96-9d83-f9c1c3b2663c, kind: Service { id: a702a2df-8aec-4a4e-9585-52df0fbccf45 }, name: Name(\"external-dns-a702a2df-8aec-4a4e-9585-52df0fbccf45\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 15334e02-b404-488e-9ce7-2a50df738630, kind: Service { id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39 }, name: Name(\"external-dns-7e2aceaf-08e1-46f2-95ec-f9032200ba39\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 88])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 28701ada-e899-402b-a2fd-f7a3fd7c41df, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 0e8e6699-5046-483c-bebf-fffb211be035, kind: Service { id: 28701ada-e899-402b-a2fd-f7a3fd7c41df }, name: Name(\"nexus-28701ada-e899-402b-a2fd-f7a3fd7c41df\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9363e23e-d709-4bd9-bca4-7fbe3ad55235, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 9e4a840d-8987-4208-af8d-b4a2771e9cc4, kind: Service { id: 9363e23e-d709-4bd9-bca4-7fbe3ad55235 }, name: Name(\"nexus-9363e23e-d709-4bd9-bca4-7fbe3ad55235\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 232, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4243c086-1bed-4413-b0c7-7c802e7bc575, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: a0dd5105-fcbd-4055-ae12-c3fb0abcf0fa, kind: Service { id: 4243c086-1bed-4413-b0c7-7c802e7bc575 }, name: Name(\"nexus-4243c086-1bed-4413-b0c7-7c802e7bc575\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 15])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 07eefccc-a16d-41be-9df2-0faa4294ae1c, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a0f87398-93b9-40a8-90b6-7059441c2401, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 2cb47b44-e1c8-4597-82f0-7ee9d5d86893, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 0e3a6bd4-4538-4ccd-a063-9c7342f19203, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f7332e25-0a75-4959-b498-faa862ee2c5f, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: d6729570-1e52-4d84-af84-f4f23623dfb6, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 1d816ece-0442-4979-9d32-7665ef85afde, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 98f9db1c-1595-4f22-a928-321363864abe, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:21.061362069Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:21.061973499Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:21.0619955Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 719635e7-d366-4895-9272-9ade9dc77ba8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:21.084367787Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 719635e7-d366-4895-9272-9ade9dc77ba8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:21.084428119Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ae10e87-0030-442b-9981-0b4c70a9c369, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:21.104761414Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2ae10e87-0030-442b-9981-0b4c70a9c369, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:21.10481788Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:21.125101238Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:21.12516329Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bf26395e-9671-4bee-b96d-1c6f76e85a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:21.144554079Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bf26395e-9671-4bee-b96d-1c6f76e85a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:21.144618981Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:21.164151295Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:21.164208337Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21544c6f-59af-4a59-a720-d40b64e76f5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:21.183170881Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 21544c6f-59af-4a59-a720-d40b64e76f5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:21.183225503Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ddb8e0b6-c193-419e-b893-def90acf2c6e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:21.202231079Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ddb8e0b6-c193-419e-b893-def90acf2c6e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:21.202305621Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a702a2df-8aec-4a4e-9585-52df0fbccf45, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:21.220894984Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a702a2df-8aec-4a4e-9585-52df0fbccf45, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:21.220950645Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:21.241027667Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:21.241088549Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:21.260034943Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:21.260089004Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:21.279203304Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:21.279280707Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:21.299858024Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:21.299918596Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:21.319534933Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:21.319616115Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7332e25-0a75-4959-b498-faa862ee2c5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:21.340131129Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f7332e25-0a75-4959-b498-faa862ee2c5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:21.340198153Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6729570-1e52-4d84-af84-f4f23623dfb6, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:21.359668724Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d6729570-1e52-4d84-af84-f4f23623dfb6, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:21.359724806Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1d816ece-0442-4979-9d32-7665ef85afde, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:21.379150476Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1d816ece-0442-4979-9d32-7665ef85afde, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:21.379210068Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 98f9db1c-1595-4f22-a928-321363864abe, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:21.398787983Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 98f9db1c-1595-4f22-a928-321363864abe, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:21.400790569Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:21.417773928Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:21.41783908Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:21.438865731Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:21.438921205Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:21.457039311Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:21.457117494Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:21.45729281Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:21.457421524Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:21.556405734Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:21.556481397Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:21.556490137Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:21.556497397Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:21.556518108Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:21.556524458Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:21.556530748Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:21.556546067Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:21.556552798Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:21.556558573Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:21.556564237Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:21.556570133Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:21.556576855Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:21.556587801Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:21.556595591Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:21.556603071Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:21.556609221Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:21.556614881Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:21.556619562Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:21.556704814Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"b0ed4a8a-bbf1-443f-a9ce-b0e20d4fe121","remote_addr":"[fd00:1122:3344:101::1]:35849","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe\", TimeNotSynchronized), (\"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2\", TimeNotSynchronized), (\"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55\", TimeNotSynchronized), (\"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893\", TimeNotSynchronized), (\"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6\", TimeNotSynchronized), (\"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde\", TimeNotSynchronized), (\"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575\", TimeNotSynchronized), (\"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45\", TimeNotSynchronized), (\"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235\", TimeNotSynchronized), (\"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401\", TimeNotSynchronized), (\"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302\", TimeNotSynchronized), (\"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203\", TimeNotSynchronized), (\"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c\", TimeNotSynchronized), (\"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17\", TimeNotSynchronized), (\"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df\", TimeNotSynchronized), (\"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7\", TimeNotSynchronized), (\"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23\", TimeNotSynchronized), (\"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f\", TimeNotSynchronized), (\"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39\", TimeNotSynchronized)]","latency_us":496273,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:21.556954842Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"3.835623236s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b0ed4a8a-bbf1-443f-a9ce-b0e20d4fe121\", \"content-length\": \"124\", \"date\": \"Thu, 15 Feb 2024 05:57:21 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b0ed4a8a-bbf1-443f-a9ce-b0e20d4fe121\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 719635e7-d366-4895-9272-9ade9dc77ba8, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2ae10e87-0030-442b-9981-0b4c70a9c369, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a62bfe46-059b-4ea1-b2a5-7f98d258138d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 44034536-a7b3-4ef5-9ea6-8cd61921d1ee, kind: Service { id: a62bfe46-059b-4ea1-b2a5-7f98d258138d }, name: Name(\"ntp-a62bfe46-059b-4ea1-b2a5-7f98d258138d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 159])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bf26395e-9671-4bee-b96d-1c6f76e85a02, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 21544c6f-59af-4a59-a720-d40b64e76f5d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: ddb8e0b6-c193-419e-b893-def90acf2c6e, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: a702a2df-8aec-4a4e-9585-52df0fbccf45, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 80b71d9a-f960-4a96-9d83-f9c1c3b2663c, kind: Service { id: a702a2df-8aec-4a4e-9585-52df0fbccf45 }, name: Name(\"external-dns-a702a2df-8aec-4a4e-9585-52df0fbccf45\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 15334e02-b404-488e-9ce7-2a50df738630, kind: Service { id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39 }, name: Name(\"external-dns-7e2aceaf-08e1-46f2-95ec-f9032200ba39\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 88])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 28701ada-e899-402b-a2fd-f7a3fd7c41df, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 0e8e6699-5046-483c-bebf-fffb211be035, kind: Service { id: 28701ada-e899-402b-a2fd-f7a3fd7c41df }, name: Name(\"nexus-28701ada-e899-402b-a2fd-f7a3fd7c41df\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9363e23e-d709-4bd9-bca4-7fbe3ad55235, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 9e4a840d-8987-4208-af8d-b4a2771e9cc4, kind: Service { id: 9363e23e-d709-4bd9-bca4-7fbe3ad55235 }, name: Name(\"nexus-9363e23e-d709-4bd9-bca4-7fbe3ad55235\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 232, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4243c086-1bed-4413-b0c7-7c802e7bc575, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: a0dd5105-fcbd-4055-ae12-c3fb0abcf0fa, kind: Service { id: 4243c086-1bed-4413-b0c7-7c802e7bc575 }, name: Name(\"nexus-4243c086-1bed-4413-b0c7-7c802e7bc575\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 15])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 07eefccc-a16d-41be-9df2-0faa4294ae1c, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a0f87398-93b9-40a8-90b6-7059441c2401, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 2cb47b44-e1c8-4597-82f0-7ee9d5d86893, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 0e3a6bd4-4538-4ccd-a063-9c7342f19203, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f7332e25-0a75-4959-b498-faa862ee2c5f, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: d6729570-1e52-4d84-af84-f4f23623dfb6, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 1d816ece-0442-4979-9d32-7665ef85afde, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 98f9db1c-1595-4f22-a928-321363864abe, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:25.383362443Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:25.383977353Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:25.383990083Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 719635e7-d366-4895-9272-9ade9dc77ba8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:25.408765929Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 719635e7-d366-4895-9272-9ade9dc77ba8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:25.411305703Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ae10e87-0030-442b-9981-0b4c70a9c369, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:25.428940154Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2ae10e87-0030-442b-9981-0b4c70a9c369, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:25.429001776Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:25.448219529Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:25.448281311Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bf26395e-9671-4bee-b96d-1c6f76e85a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:25.467561376Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bf26395e-9671-4bee-b96d-1c6f76e85a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:25.467620518Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:25.487293086Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:25.487351088Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21544c6f-59af-4a59-a720-d40b64e76f5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:25.505984731Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 21544c6f-59af-4a59-a720-d40b64e76f5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:25.508402401Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ddb8e0b6-c193-419e-b893-def90acf2c6e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:25.525274647Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ddb8e0b6-c193-419e-b893-def90acf2c6e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:25.525340349Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a702a2df-8aec-4a4e-9585-52df0fbccf45, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:25.543840108Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a702a2df-8aec-4a4e-9585-52df0fbccf45, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:25.54389891Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:25.562778092Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:25.562842304Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:25.582045417Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:25.582112849Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:25.602037915Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:25.602105297Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:25.621853248Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:25.62191667Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:25.642084634Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:25.642148436Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7332e25-0a75-4959-b498-faa862ee2c5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:25.663073445Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f7332e25-0a75-4959-b498-faa862ee2c5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:25.663134067Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6729570-1e52-4d84-af84-f4f23623dfb6, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:25.682338127Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d6729570-1e52-4d84-af84-f4f23623dfb6, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:25.682395232Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1d816ece-0442-4979-9d32-7665ef85afde, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:25.701947216Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1d816ece-0442-4979-9d32-7665ef85afde, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:25.702003698Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 98f9db1c-1595-4f22-a928-321363864abe, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:25.720997584Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 98f9db1c-1595-4f22-a928-321363864abe, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:25.721062766Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:25.739365169Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:25.73942323Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:25.758301482Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:25.758358164Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:25.776992858Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:25.777065094Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:25.777235086Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:25.77734948Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:25.87087006Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:25.870970913Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:25.870979093Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:25.870985754Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:25.870992674Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:25.871000484Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:25.871014415Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:25.871020475Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:25.871027835Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:25.871035125Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:25.871042586Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:25.871048936Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:25.871055476Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:25.871063106Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:25.871067996Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:25.871074056Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:25.871082957Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:25.871089247Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:25.871094137Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:25.871159349Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"f5c3d8ee-15a6-4844-8682-2f01249e1c88","remote_addr":"[fd00:1122:3344:101::1]:35849","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39\", TimeNotSynchronized), (\"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235\", TimeNotSynchronized), (\"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302\", TimeNotSynchronized), (\"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401\", TimeNotSynchronized), (\"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45\", TimeNotSynchronized), (\"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17\", TimeNotSynchronized), (\"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7\", TimeNotSynchronized), (\"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893\", TimeNotSynchronized), (\"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c\", TimeNotSynchronized), (\"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f\", TimeNotSynchronized), (\"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6\", TimeNotSynchronized), (\"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde\", TimeNotSynchronized), (\"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575\", TimeNotSynchronized), (\"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe\", TimeNotSynchronized), (\"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23\", TimeNotSynchronized), (\"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55\", TimeNotSynchronized), (\"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203\", TimeNotSynchronized), (\"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df\", TimeNotSynchronized), (\"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2\", TimeNotSynchronized)]","latency_us":488718,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:25.871397197Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"2.795902409s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f5c3d8ee-15a6-4844-8682-2f01249e1c88\", \"content-length\": \"124\", \"date\": \"Thu, 15 Feb 2024 05:57:25 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f5c3d8ee-15a6-4844-8682-2f01249e1c88\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:27.598070539Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:47004"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:27.598156049Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"4ea02e78-3e62-4925-a5e0-d6eb01583464","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:47004","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":48,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 719635e7-d366-4895-9272-9ade9dc77ba8, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2ae10e87-0030-442b-9981-0b4c70a9c369, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a62bfe46-059b-4ea1-b2a5-7f98d258138d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 44034536-a7b3-4ef5-9ea6-8cd61921d1ee, kind: Service { id: a62bfe46-059b-4ea1-b2a5-7f98d258138d }, name: Name(\"ntp-a62bfe46-059b-4ea1-b2a5-7f98d258138d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 159])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bf26395e-9671-4bee-b96d-1c6f76e85a02, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 21544c6f-59af-4a59-a720-d40b64e76f5d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: ddb8e0b6-c193-419e-b893-def90acf2c6e, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: a702a2df-8aec-4a4e-9585-52df0fbccf45, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 80b71d9a-f960-4a96-9d83-f9c1c3b2663c, kind: Service { id: a702a2df-8aec-4a4e-9585-52df0fbccf45 }, name: Name(\"external-dns-a702a2df-8aec-4a4e-9585-52df0fbccf45\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 15334e02-b404-488e-9ce7-2a50df738630, kind: Service { id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39 }, name: Name(\"external-dns-7e2aceaf-08e1-46f2-95ec-f9032200ba39\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 88])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 28701ada-e899-402b-a2fd-f7a3fd7c41df, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 0e8e6699-5046-483c-bebf-fffb211be035, kind: Service { id: 28701ada-e899-402b-a2fd-f7a3fd7c41df }, name: Name(\"nexus-28701ada-e899-402b-a2fd-f7a3fd7c41df\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9363e23e-d709-4bd9-bca4-7fbe3ad55235, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 9e4a840d-8987-4208-af8d-b4a2771e9cc4, kind: Service { id: 9363e23e-d709-4bd9-bca4-7fbe3ad55235 }, name: Name(\"nexus-9363e23e-d709-4bd9-bca4-7fbe3ad55235\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 232, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4243c086-1bed-4413-b0c7-7c802e7bc575, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: a0dd5105-fcbd-4055-ae12-c3fb0abcf0fa, kind: Service { id: 4243c086-1bed-4413-b0c7-7c802e7bc575 }, name: Name(\"nexus-4243c086-1bed-4413-b0c7-7c802e7bc575\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 15])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 07eefccc-a16d-41be-9df2-0faa4294ae1c, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a0f87398-93b9-40a8-90b6-7059441c2401, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 2cb47b44-e1c8-4597-82f0-7ee9d5d86893, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 0e3a6bd4-4538-4ccd-a063-9c7342f19203, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f7332e25-0a75-4959-b498-faa862ee2c5f, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: d6729570-1e52-4d84-af84-f4f23623dfb6, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 1d816ece-0442-4979-9d32-7665ef85afde, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 98f9db1c-1595-4f22-a928-321363864abe, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:28.66104073Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:28.661779445Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:28.661791065Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 719635e7-d366-4895-9272-9ade9dc77ba8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:28.684265075Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 719635e7-d366-4895-9272-9ade9dc77ba8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:28.684331017Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ae10e87-0030-442b-9981-0b4c70a9c369, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:28.704570074Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2ae10e87-0030-442b-9981-0b4c70a9c369, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:28.704630966Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:28.723735155Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:28.723794457Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bf26395e-9671-4bee-b96d-1c6f76e85a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:28.744317443Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bf26395e-9671-4bee-b96d-1c6f76e85a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:28.744382546Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:28.763879097Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:28.76393903Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21544c6f-59af-4a59-a720-d40b64e76f5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:28.783881676Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 21544c6f-59af-4a59-a720-d40b64e76f5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:28.783936206Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ddb8e0b6-c193-419e-b893-def90acf2c6e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:28.802974245Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ddb8e0b6-c193-419e-b893-def90acf2c6e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:28.803038127Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a702a2df-8aec-4a4e-9585-52df0fbccf45, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:28.821971841Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a702a2df-8aec-4a4e-9585-52df0fbccf45, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:28.822031273Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:28.841239646Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:28.841302358Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:28.860789749Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:28.860854962Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:28.879679574Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:28.879739004Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:28.899001678Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:28.899085311Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:28.91788402Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:28.917948032Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7332e25-0a75-4959-b498-faa862ee2c5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:28.937397173Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f7332e25-0a75-4959-b498-faa862ee2c5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:28.937456735Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6729570-1e52-4d84-af84-f4f23623dfb6, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:28.957084121Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d6729570-1e52-4d84-af84-f4f23623dfb6, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:28.957141603Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1d816ece-0442-4979-9d32-7665ef85afde, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:28.976657406Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1d816ece-0442-4979-9d32-7665ef85afde, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:28.976717158Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 98f9db1c-1595-4f22-a928-321363864abe, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:28.998051421Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 98f9db1c-1595-4f22-a928-321363864abe, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:28.998107842Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:29.019562509Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:29.019621231Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:29.041527922Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:29.041588164Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:29.061722798Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:29.06179802Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:29.061968056Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:29.0620979Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:29.159663724Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:29.159750576Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:29.159758196Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:29.159766887Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:29.159773247Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:29.159779377Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:29.159785937Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:29.159798638Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:29.159806758Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:29.159814369Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:29.159829739Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:29.159837369Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:29.159843829Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:29.15985163Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:29.15986007Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:29.1598659Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:29.15987055Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:29.159876562Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:29.159881728Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:29.159961893Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"3fcd2830-3683-4b67-b57e-6f22346702ad","remote_addr":"[fd00:1122:3344:101::1]:35849","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe\", TimeNotSynchronized), (\"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c\", TimeNotSynchronized), (\"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2\", TimeNotSynchronized), (\"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302\", TimeNotSynchronized), (\"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df\", TimeNotSynchronized), (\"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575\", TimeNotSynchronized), (\"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde\", TimeNotSynchronized), (\"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203\", TimeNotSynchronized), (\"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23\", TimeNotSynchronized), (\"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7\", TimeNotSynchronized), (\"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893\", TimeNotSynchronized), (\"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45\", TimeNotSynchronized), (\"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39\", TimeNotSynchronized), (\"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f\", TimeNotSynchronized), (\"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6\", TimeNotSynchronized), (\"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401\", TimeNotSynchronized), (\"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235\", TimeNotSynchronized), (\"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55\", TimeNotSynchronized), (\"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17\", TimeNotSynchronized)]","latency_us":499738,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:29.160193511Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"4.384451752s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3fcd2830-3683-4b67-b57e-6f22346702ad\", \"content-length\": \"124\", \"date\": \"Thu, 15 Feb 2024 05:57:28 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3fcd2830-3683-4b67-b57e-6f22346702ad\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 719635e7-d366-4895-9272-9ade9dc77ba8, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2ae10e87-0030-442b-9981-0b4c70a9c369, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a62bfe46-059b-4ea1-b2a5-7f98d258138d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 44034536-a7b3-4ef5-9ea6-8cd61921d1ee, kind: Service { id: a62bfe46-059b-4ea1-b2a5-7f98d258138d }, name: Name(\"ntp-a62bfe46-059b-4ea1-b2a5-7f98d258138d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 159])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bf26395e-9671-4bee-b96d-1c6f76e85a02, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 21544c6f-59af-4a59-a720-d40b64e76f5d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: ddb8e0b6-c193-419e-b893-def90acf2c6e, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: a702a2df-8aec-4a4e-9585-52df0fbccf45, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 80b71d9a-f960-4a96-9d83-f9c1c3b2663c, kind: Service { id: a702a2df-8aec-4a4e-9585-52df0fbccf45 }, name: Name(\"external-dns-a702a2df-8aec-4a4e-9585-52df0fbccf45\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 15334e02-b404-488e-9ce7-2a50df738630, kind: Service { id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39 }, name: Name(\"external-dns-7e2aceaf-08e1-46f2-95ec-f9032200ba39\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 88])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 28701ada-e899-402b-a2fd-f7a3fd7c41df, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 0e8e6699-5046-483c-bebf-fffb211be035, kind: Service { id: 28701ada-e899-402b-a2fd-f7a3fd7c41df }, name: Name(\"nexus-28701ada-e899-402b-a2fd-f7a3fd7c41df\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9363e23e-d709-4bd9-bca4-7fbe3ad55235, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 9e4a840d-8987-4208-af8d-b4a2771e9cc4, kind: Service { id: 9363e23e-d709-4bd9-bca4-7fbe3ad55235 }, name: Name(\"nexus-9363e23e-d709-4bd9-bca4-7fbe3ad55235\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 232, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4243c086-1bed-4413-b0c7-7c802e7bc575, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: a0dd5105-fcbd-4055-ae12-c3fb0abcf0fa, kind: Service { id: 4243c086-1bed-4413-b0c7-7c802e7bc575 }, name: Name(\"nexus-4243c086-1bed-4413-b0c7-7c802e7bc575\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 15])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 07eefccc-a16d-41be-9df2-0faa4294ae1c, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a0f87398-93b9-40a8-90b6-7059441c2401, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 2cb47b44-e1c8-4597-82f0-7ee9d5d86893, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 0e3a6bd4-4538-4ccd-a063-9c7342f19203, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f7332e25-0a75-4959-b498-faa862ee2c5f, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: d6729570-1e52-4d84-af84-f4f23623dfb6, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 1d816ece-0442-4979-9d32-7665ef85afde, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 98f9db1c-1595-4f22-a928-321363864abe, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:33.533175544Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:33.533809535Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:33.533828676Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 719635e7-d366-4895-9272-9ade9dc77ba8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:33.556517063Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 719635e7-d366-4895-9272-9ade9dc77ba8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:33.556587975Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ae10e87-0030-442b-9981-0b4c70a9c369, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:33.573678928Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2ae10e87-0030-442b-9981-0b4c70a9c369, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:33.57373676Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:33.592852544Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:33.592914122Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bf26395e-9671-4bee-b96d-1c6f76e85a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:33.611527895Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bf26395e-9671-4bee-b96d-1c6f76e85a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:33.611599337Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:33.62960279Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:33.629660132Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21544c6f-59af-4a59-a720-d40b64e76f5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:33.647847851Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 21544c6f-59af-4a59-a720-d40b64e76f5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:33.647915874Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ddb8e0b6-c193-419e-b893-def90acf2c6e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:33.66864092Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ddb8e0b6-c193-419e-b893-def90acf2c6e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:33.668697438Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a702a2df-8aec-4a4e-9585-52df0fbccf45, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:33.687219678Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a702a2df-8aec-4a4e-9585-52df0fbccf45, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:33.68727831Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:33.705128238Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:33.707485605Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:33.723579416Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:33.723630717Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:33.741695372Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:33.741778875Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:33.759775658Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:33.7598503Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:33.778781624Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:33.778837356Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7332e25-0a75-4959-b498-faa862ee2c5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:33.796800057Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f7332e25-0a75-4959-b498-faa862ee2c5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:33.796854679Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6729570-1e52-4d84-af84-f4f23623dfb6, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:33.814820231Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d6729570-1e52-4d84-af84-f4f23623dfb6, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:33.814896903Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1d816ece-0442-4979-9d32-7665ef85afde, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:33.832667369Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1d816ece-0442-4979-9d32-7665ef85afde, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:33.83271579Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 98f9db1c-1595-4f22-a928-321363864abe, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:33.851955474Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 98f9db1c-1595-4f22-a928-321363864abe, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:33.854486087Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:33.872325745Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:33.872388387Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:33.892152018Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:33.89221475Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:33.910368858Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:33.910453871Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:33.910641877Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:33.910782172Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:34.007461986Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:34.007525538Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:34.007533698Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:34.007548579Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:34.007562709Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:34.007569189Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:34.007576989Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:34.00759333Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:34.00759998Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:34.00760759Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:34.007614851Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:34.007620991Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:34.007625511Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:34.007631491Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:34.007636911Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:34.007641112Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:34.007646552Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:34.007651092Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:34.007656522Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:34.007718726Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"6529ff67-a885-4c4f-b518-626d87897568","remote_addr":"[fd00:1122:3344:101::1]:35849","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302\", TimeNotSynchronized), (\"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f\", TimeNotSynchronized), (\"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401\", TimeNotSynchronized), (\"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df\", TimeNotSynchronized), (\"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55\", TimeNotSynchronized), (\"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c\", TimeNotSynchronized), (\"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17\", TimeNotSynchronized), (\"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575\", TimeNotSynchronized), (\"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45\", TimeNotSynchronized), (\"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203\", TimeNotSynchronized), (\"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2\", TimeNotSynchronized), (\"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7\", TimeNotSynchronized), (\"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39\", TimeNotSynchronized), (\"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6\", TimeNotSynchronized), (\"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe\", TimeNotSynchronized), (\"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235\", TimeNotSynchronized), (\"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893\", TimeNotSynchronized), (\"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23\", TimeNotSynchronized), (\"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde\", TimeNotSynchronized)]","latency_us":475397,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:34.007944802Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"4.272178648s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6529ff67-a885-4c4f-b518-626d87897568\", \"content-length\": \"124\", \"date\": \"Thu, 15 Feb 2024 05:57:33 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6529ff67-a885-4c4f-b518-626d87897568\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 719635e7-d366-4895-9272-9ade9dc77ba8, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2ae10e87-0030-442b-9981-0b4c70a9c369, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a62bfe46-059b-4ea1-b2a5-7f98d258138d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 44034536-a7b3-4ef5-9ea6-8cd61921d1ee, kind: Service { id: a62bfe46-059b-4ea1-b2a5-7f98d258138d }, name: Name(\"ntp-a62bfe46-059b-4ea1-b2a5-7f98d258138d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 159])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bf26395e-9671-4bee-b96d-1c6f76e85a02, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 21544c6f-59af-4a59-a720-d40b64e76f5d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: ddb8e0b6-c193-419e-b893-def90acf2c6e, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: a702a2df-8aec-4a4e-9585-52df0fbccf45, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 80b71d9a-f960-4a96-9d83-f9c1c3b2663c, kind: Service { id: a702a2df-8aec-4a4e-9585-52df0fbccf45 }, name: Name(\"external-dns-a702a2df-8aec-4a4e-9585-52df0fbccf45\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 15334e02-b404-488e-9ce7-2a50df738630, kind: Service { id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39 }, name: Name(\"external-dns-7e2aceaf-08e1-46f2-95ec-f9032200ba39\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 88])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 28701ada-e899-402b-a2fd-f7a3fd7c41df, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 0e8e6699-5046-483c-bebf-fffb211be035, kind: Service { id: 28701ada-e899-402b-a2fd-f7a3fd7c41df }, name: Name(\"nexus-28701ada-e899-402b-a2fd-f7a3fd7c41df\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9363e23e-d709-4bd9-bca4-7fbe3ad55235, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 9e4a840d-8987-4208-af8d-b4a2771e9cc4, kind: Service { id: 9363e23e-d709-4bd9-bca4-7fbe3ad55235 }, name: Name(\"nexus-9363e23e-d709-4bd9-bca4-7fbe3ad55235\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 232, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4243c086-1bed-4413-b0c7-7c802e7bc575, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: a0dd5105-fcbd-4055-ae12-c3fb0abcf0fa, kind: Service { id: 4243c086-1bed-4413-b0c7-7c802e7bc575 }, name: Name(\"nexus-4243c086-1bed-4413-b0c7-7c802e7bc575\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 15])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 07eefccc-a16d-41be-9df2-0faa4294ae1c, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a0f87398-93b9-40a8-90b6-7059441c2401, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 2cb47b44-e1c8-4597-82f0-7ee9d5d86893, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 0e3a6bd4-4538-4ccd-a063-9c7342f19203, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f7332e25-0a75-4959-b498-faa862ee2c5f, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: d6729570-1e52-4d84-af84-f4f23623dfb6, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 1d816ece-0442-4979-9d32-7665ef85afde, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 98f9db1c-1595-4f22-a928-321363864abe, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:38.26963012Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:38.270370644Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:38.270383454Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 719635e7-d366-4895-9272-9ade9dc77ba8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:38.292306777Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 719635e7-d366-4895-9272-9ade9dc77ba8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:38.292375809Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ae10e87-0030-442b-9981-0b4c70a9c369, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:38.312913955Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2ae10e87-0030-442b-9981-0b4c70a9c369, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:38.312975517Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:38.334213177Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:38.334283759Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bf26395e-9671-4bee-b96d-1c6f76e85a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:38.354748853Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bf26395e-9671-4bee-b96d-1c6f76e85a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:38.354818796Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:38.374035579Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:38.374109731Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21544c6f-59af-4a59-a720-d40b64e76f5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:38.392376993Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 21544c6f-59af-4a59-a720-d40b64e76f5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:38.392435485Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ddb8e0b6-c193-419e-b893-def90acf2c6e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:38.410870052Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ddb8e0b6-c193-419e-b893-def90acf2c6e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:38.410942264Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a702a2df-8aec-4a4e-9585-52df0fbccf45, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:38.429379391Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a702a2df-8aec-4a4e-9585-52df0fbccf45, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:38.429432223Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:38.448061597Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:38.448115609Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:38.466855666Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:38.466911768Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:38.4851811Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:38.485231511Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:38.50350626Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:38.503560375Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:38.521297899Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:38.521368661Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7332e25-0a75-4959-b498-faa862ee2c5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:38.539903992Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f7332e25-0a75-4959-b498-faa862ee2c5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:38.539961944Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6729570-1e52-4d84-af84-f4f23623dfb6, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:38.559044322Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d6729570-1e52-4d84-af84-f4f23623dfb6, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:38.559108995Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1d816ece-0442-4979-9d32-7665ef85afde, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:38.577128848Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1d816ece-0442-4979-9d32-7665ef85afde, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:38.577176999Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 98f9db1c-1595-4f22-a928-321363864abe, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:38.59601763Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 98f9db1c-1595-4f22-a928-321363864abe, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:38.596082522Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:38.614681575Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:38.617092154Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:38.633107152Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:38.633167895Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:38.652307594Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:38.652383847Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:38.652577293Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:38.652721168Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:38.745961109Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:38.746044262Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:38.746053372Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:38.746061942Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:38.746096783Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:38.746107964Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:38.746118514Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:38.746129024Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:38.746140135Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:38.746149815Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:38.746160625Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:38.746171016Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:38.746187306Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:38.746195757Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:38.746202017Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:38.746215667Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:38.746224427Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:38.746232738Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:38.746240998Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:38.746321761Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"a9897d75-0a5d-4aea-851e-59b43d3c32f6","remote_addr":"[fd00:1122:3344:101::1]:35849","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401\", TimeNotSynchronized), (\"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203\", TimeNotSynchronized), (\"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2\", TimeNotSynchronized), (\"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe\", TimeNotSynchronized), (\"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde\", TimeNotSynchronized), (\"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302\", TimeNotSynchronized), (\"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45\", TimeNotSynchronized), (\"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23\", TimeNotSynchronized), (\"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55\", TimeNotSynchronized), (\"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f\", TimeNotSynchronized), (\"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7\", TimeNotSynchronized), (\"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235\", TimeNotSynchronized), (\"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575\", TimeNotSynchronized), (\"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c\", TimeNotSynchronized), (\"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39\", TimeNotSynchronized), (\"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df\", TimeNotSynchronized), (\"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893\", TimeNotSynchronized), (\"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17\", TimeNotSynchronized), (\"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6\", TimeNotSynchronized)]","latency_us":477479,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:38.746559958Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"6.075585201s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a9897d75-0a5d-4aea-851e-59b43d3c32f6\", \"content-length\": \"124\", \"date\": \"Thu, 15 Feb 2024 05:57:38 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a9897d75-0a5d-4aea-851e-59b43d3c32f6\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 719635e7-d366-4895-9272-9ade9dc77ba8, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2ae10e87-0030-442b-9981-0b4c70a9c369, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a62bfe46-059b-4ea1-b2a5-7f98d258138d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 44034536-a7b3-4ef5-9ea6-8cd61921d1ee, kind: Service { id: a62bfe46-059b-4ea1-b2a5-7f98d258138d }, name: Name(\"ntp-a62bfe46-059b-4ea1-b2a5-7f98d258138d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 159])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bf26395e-9671-4bee-b96d-1c6f76e85a02, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 21544c6f-59af-4a59-a720-d40b64e76f5d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: ddb8e0b6-c193-419e-b893-def90acf2c6e, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: a702a2df-8aec-4a4e-9585-52df0fbccf45, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 80b71d9a-f960-4a96-9d83-f9c1c3b2663c, kind: Service { id: a702a2df-8aec-4a4e-9585-52df0fbccf45 }, name: Name(\"external-dns-a702a2df-8aec-4a4e-9585-52df0fbccf45\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 15334e02-b404-488e-9ce7-2a50df738630, kind: Service { id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39 }, name: Name(\"external-dns-7e2aceaf-08e1-46f2-95ec-f9032200ba39\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 88])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 28701ada-e899-402b-a2fd-f7a3fd7c41df, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 0e8e6699-5046-483c-bebf-fffb211be035, kind: Service { id: 28701ada-e899-402b-a2fd-f7a3fd7c41df }, name: Name(\"nexus-28701ada-e899-402b-a2fd-f7a3fd7c41df\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9363e23e-d709-4bd9-bca4-7fbe3ad55235, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 9e4a840d-8987-4208-af8d-b4a2771e9cc4, kind: Service { id: 9363e23e-d709-4bd9-bca4-7fbe3ad55235 }, name: Name(\"nexus-9363e23e-d709-4bd9-bca4-7fbe3ad55235\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 232, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4243c086-1bed-4413-b0c7-7c802e7bc575, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: a0dd5105-fcbd-4055-ae12-c3fb0abcf0fa, kind: Service { id: 4243c086-1bed-4413-b0c7-7c802e7bc575 }, name: Name(\"nexus-4243c086-1bed-4413-b0c7-7c802e7bc575\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 15])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 07eefccc-a16d-41be-9df2-0faa4294ae1c, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a0f87398-93b9-40a8-90b6-7059441c2401, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 2cb47b44-e1c8-4597-82f0-7ee9d5d86893, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 0e3a6bd4-4538-4ccd-a063-9c7342f19203, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f7332e25-0a75-4959-b498-faa862ee2c5f, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: d6729570-1e52-4d84-af84-f4f23623dfb6, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 1d816ece-0442-4979-9d32-7665ef85afde, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 98f9db1c-1595-4f22-a928-321363864abe, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:44.807248709Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:44.807955813Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:44.807968093Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 719635e7-d366-4895-9272-9ade9dc77ba8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:44.832296123Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 719635e7-d366-4895-9272-9ade9dc77ba8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:44.834872149Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ae10e87-0030-442b-9981-0b4c70a9c369, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:44.854705092Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2ae10e87-0030-442b-9981-0b4c70a9c369, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:44.854760354Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:44.875808277Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:44.877888466Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bf26395e-9671-4bee-b96d-1c6f76e85a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:44.895353821Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bf26395e-9671-4bee-b96d-1c6f76e85a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:44.895410123Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:44.914490702Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:44.914567374Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21544c6f-59af-4a59-a720-d40b64e76f5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:44.933097594Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 21544c6f-59af-4a59-a720-d40b64e76f5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:44.933174877Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ddb8e0b6-c193-419e-b893-def90acf2c6e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:44.951582583Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ddb8e0b6-c193-419e-b893-def90acf2c6e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:44.951640635Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a702a2df-8aec-4a4e-9585-52df0fbccf45, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:44.969855875Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a702a2df-8aec-4a4e-9585-52df0fbccf45, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:44.969915147Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:44.988517589Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:44.991081154Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:45.007743273Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:45.010184203Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:45.026148099Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:45.026213241Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:45.046573372Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:45.049000752Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:45.066276421Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:45.066344763Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7332e25-0a75-4959-b498-faa862ee2c5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:45.08479042Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f7332e25-0a75-4959-b498-faa862ee2c5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:45.084864533Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6729570-1e52-4d84-af84-f4f23623dfb6, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:45.102951599Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d6729570-1e52-4d84-af84-f4f23623dfb6, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:45.103012841Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1d816ece-0442-4979-9d32-7665ef85afde, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:45.120877329Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1d816ece-0442-4979-9d32-7665ef85afde, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:45.120935651Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 98f9db1c-1595-4f22-a928-321363864abe, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:45.140835777Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 98f9db1c-1595-4f22-a928-321363864abe, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:45.140906639Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:45.160122302Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:45.160182224Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:45.1785772Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:45.178634151Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:45.197229794Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:45.197317091Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:45.197487843Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:45.197623317Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:45.29061877Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:45.290692312Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:45.290698863Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:45.290705873Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:45.290713473Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:45.290722173Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:45.290727934Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:45.290735784Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:45.290743314Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:45.290752384Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:45.290759184Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:45.290767835Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:45.290776745Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:45.290783795Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:45.290790606Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:45.290799486Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:45.290818337Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:45.290827507Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:45.290836477Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:45.290956321Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"82bfd3bc-908b-41b7-8dde-22c67caddc84","remote_addr":"[fd00:1122:3344:101::1]:35849","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6\", TimeNotSynchronized), (\"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde\", TimeNotSynchronized), (\"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7\", TimeNotSynchronized), (\"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe\", TimeNotSynchronized), (\"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39\", TimeNotSynchronized), (\"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575\", TimeNotSynchronized), (\"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17\", TimeNotSynchronized), (\"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2\", TimeNotSynchronized), (\"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df\", TimeNotSynchronized), (\"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302\", TimeNotSynchronized), (\"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f\", TimeNotSynchronized), (\"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45\", TimeNotSynchronized), (\"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c\", TimeNotSynchronized), (\"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203\", TimeNotSynchronized), (\"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23\", TimeNotSynchronized), (\"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235\", TimeNotSynchronized), (\"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401\", TimeNotSynchronized), (\"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55\", TimeNotSynchronized), (\"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893\", TimeNotSynchronized)]","latency_us":484533,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:45.291212919Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"9.410252324s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"82bfd3bc-908b-41b7-8dde-22c67caddc84\", \"content-length\": \"124\", \"date\": \"Thu, 15 Feb 2024 05:57:44 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"82bfd3bc-908b-41b7-8dde-22c67caddc84\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 719635e7-d366-4895-9272-9ade9dc77ba8, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2ae10e87-0030-442b-9981-0b4c70a9c369, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a62bfe46-059b-4ea1-b2a5-7f98d258138d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 44034536-a7b3-4ef5-9ea6-8cd61921d1ee, kind: Service { id: a62bfe46-059b-4ea1-b2a5-7f98d258138d }, name: Name(\"ntp-a62bfe46-059b-4ea1-b2a5-7f98d258138d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 159])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bf26395e-9671-4bee-b96d-1c6f76e85a02, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 21544c6f-59af-4a59-a720-d40b64e76f5d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: ddb8e0b6-c193-419e-b893-def90acf2c6e, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: a702a2df-8aec-4a4e-9585-52df0fbccf45, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 80b71d9a-f960-4a96-9d83-f9c1c3b2663c, kind: Service { id: a702a2df-8aec-4a4e-9585-52df0fbccf45 }, name: Name(\"external-dns-a702a2df-8aec-4a4e-9585-52df0fbccf45\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 15334e02-b404-488e-9ce7-2a50df738630, kind: Service { id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39 }, name: Name(\"external-dns-7e2aceaf-08e1-46f2-95ec-f9032200ba39\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 88])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 28701ada-e899-402b-a2fd-f7a3fd7c41df, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 0e8e6699-5046-483c-bebf-fffb211be035, kind: Service { id: 28701ada-e899-402b-a2fd-f7a3fd7c41df }, name: Name(\"nexus-28701ada-e899-402b-a2fd-f7a3fd7c41df\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9363e23e-d709-4bd9-bca4-7fbe3ad55235, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 9e4a840d-8987-4208-af8d-b4a2771e9cc4, kind: Service { id: 9363e23e-d709-4bd9-bca4-7fbe3ad55235 }, name: Name(\"nexus-9363e23e-d709-4bd9-bca4-7fbe3ad55235\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 232, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4243c086-1bed-4413-b0c7-7c802e7bc575, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: a0dd5105-fcbd-4055-ae12-c3fb0abcf0fa, kind: Service { id: 4243c086-1bed-4413-b0c7-7c802e7bc575 }, name: Name(\"nexus-4243c086-1bed-4413-b0c7-7c802e7bc575\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 15])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 07eefccc-a16d-41be-9df2-0faa4294ae1c, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a0f87398-93b9-40a8-90b6-7059441c2401, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 2cb47b44-e1c8-4597-82f0-7ee9d5d86893, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 0e3a6bd4-4538-4ccd-a063-9c7342f19203, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f7332e25-0a75-4959-b498-faa862ee2c5f, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: d6729570-1e52-4d84-af84-f4f23623dfb6, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 1d816ece-0442-4979-9d32-7665ef85afde, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 98f9db1c-1595-4f22-a928-321363864abe, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:54.677177395Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:54.677878288Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:54.677897649Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 719635e7-d366-4895-9272-9ade9dc77ba8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:54.698290821Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 719635e7-d366-4895-9272-9ade9dc77ba8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:54.698355323Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ae10e87-0030-442b-9981-0b4c70a9c369, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:54.720635057Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2ae10e87-0030-442b-9981-0b4c70a9c369, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:54.72071919Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:54.74169397Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:54.741767723Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bf26395e-9671-4bee-b96d-1c6f76e85a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:54.763611082Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bf26395e-9671-4bee-b96d-1c6f76e85a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:54.763679605Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:54.785268956Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:54.785330098Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21544c6f-59af-4a59-a720-d40b64e76f5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:54.810380623Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 21544c6f-59af-4a59-a720-d40b64e76f5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:54.810449035Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ddb8e0b6-c193-419e-b893-def90acf2c6e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:54.834138153Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ddb8e0b6-c193-419e-b893-def90acf2c6e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:54.834211608Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a702a2df-8aec-4a4e-9585-52df0fbccf45, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:54.857298108Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a702a2df-8aec-4a4e-9585-52df0fbccf45, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:54.85736391Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:54.881261427Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:54.883986988Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:54.903324864Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:54.903383776Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:54.926560069Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:54.926621611Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:54.947281442Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:54.947344944Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:54.970109004Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:54.970187576Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7332e25-0a75-4959-b498-faa862ee2c5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:54.99269037Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f7332e25-0a75-4959-b498-faa862ee2c5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:54.992753709Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6729570-1e52-4d84-af84-f4f23623dfb6, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:55.016203812Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d6729570-1e52-4d84-af84-f4f23623dfb6, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:55.016274034Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1d816ece-0442-4979-9d32-7665ef85afde, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:55.038734434Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1d816ece-0442-4979-9d32-7665ef85afde, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:55.038792846Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 98f9db1c-1595-4f22-a928-321363864abe, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:55.061195834Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 98f9db1c-1595-4f22-a928-321363864abe, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:55.061257156Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:55.085521945Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:55.085582917Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:55.10843832Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:55.108499682Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:55.130826257Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:55.13091165Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:55.131073265Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:55.131186369Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:55.240877422Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:55.240950964Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:55.240959065Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:55.240964245Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:55.240970545Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:55.240976915Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:55.240982915Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:55.240989186Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:55.240995056Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:55.241001256Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:55.241007366Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:55.241020189Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:55.241025814Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:55.241048624Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:55.241054671Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:55.241058978Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:55.241064688Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:55.241071798Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:55.241077738Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2770","zone":"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:55.241230814Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"41c4ebd4-0e1b-4a27-b538-3a5c4ef1d909","remote_addr":"[fd00:1122:3344:101::1]:35849","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39\", TimeNotSynchronized), (\"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df\", TimeNotSynchronized), (\"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235\", TimeNotSynchronized), (\"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401\", TimeNotSynchronized), (\"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2\", TimeNotSynchronized), (\"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7\", TimeNotSynchronized), (\"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde\", TimeNotSynchronized), (\"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17\", TimeNotSynchronized), (\"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893\", TimeNotSynchronized), (\"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302\", TimeNotSynchronized), (\"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6\", TimeNotSynchronized), (\"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe\", TimeNotSynchronized), (\"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f\", TimeNotSynchronized), (\"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575\", TimeNotSynchronized), (\"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203\", TimeNotSynchronized), (\"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23\", TimeNotSynchronized), (\"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55\", TimeNotSynchronized), (\"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45\", TimeNotSynchronized), (\"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c\", TimeNotSynchronized)]","latency_us":565072,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:57:55.241505052Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:337","retry_after":"9.985980022s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"41c4ebd4-0e1b-4a27-b538-3a5c4ef1d909\", \"content-length\": \"124\", \"date\": \"Thu, 15 Feb 2024 05:57:54 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"41c4ebd4-0e1b-4a27-b538-3a5c4ef1d909\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:57.549610685Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:46968"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:57:57.549675837Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"04e6c94c-089f-47f4-a911-64507bff02aa","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:46968","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":38,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 719635e7-d366-4895-9272-9ade9dc77ba8, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2ae10e87-0030-442b-9981-0b4c70a9c369, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a62bfe46-059b-4ea1-b2a5-7f98d258138d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 44034536-a7b3-4ef5-9ea6-8cd61921d1ee, kind: Service { id: a62bfe46-059b-4ea1-b2a5-7f98d258138d }, name: Name(\"ntp-a62bfe46-059b-4ea1-b2a5-7f98d258138d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 159])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: bf26395e-9671-4bee-b96d-1c6f76e85a02, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: 21544c6f-59af-4a59-a720-d40b64e76f5d, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: ddb8e0b6-c193-419e-b893-def90acf2c6e, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: a702a2df-8aec-4a4e-9585-52df0fbccf45, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 80b71d9a-f960-4a96-9d83-f9c1c3b2663c, kind: Service { id: a702a2df-8aec-4a4e-9585-52df0fbccf45 }, name: Name(\"external-dns-a702a2df-8aec-4a4e-9585-52df0fbccf45\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 15334e02-b404-488e-9ce7-2a50df738630, kind: Service { id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39 }, name: Name(\"external-dns-7e2aceaf-08e1-46f2-95ec-f9032200ba39\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 88])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 28701ada-e899-402b-a2fd-f7a3fd7c41df, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 0e8e6699-5046-483c-bebf-fffb211be035, kind: Service { id: 28701ada-e899-402b-a2fd-f7a3fd7c41df }, name: Name(\"nexus-28701ada-e899-402b-a2fd-f7a3fd7c41df\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9363e23e-d709-4bd9-bca4-7fbe3ad55235, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 9e4a840d-8987-4208-af8d-b4a2771e9cc4, kind: Service { id: 9363e23e-d709-4bd9-bca4-7fbe3ad55235 }, name: Name(\"nexus-9363e23e-d709-4bd9-bca4-7fbe3ad55235\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 232, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4243c086-1bed-4413-b0c7-7c802e7bc575, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: a0dd5105-fcbd-4055-ae12-c3fb0abcf0fa, kind: Service { id: 4243c086-1bed-4413-b0c7-7c802e7bc575 }, name: Name(\"nexus-4243c086-1bed-4413-b0c7-7c802e7bc575\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 15])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 07eefccc-a16d-41be-9df2-0faa4294ae1c, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: a0f87398-93b9-40a8-90b6-7059441c2401, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 2cb47b44-e1c8-4597-82f0-7ee9d5d86893, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 0e3a6bd4-4538-4ccd-a063-9c7342f19203, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External } } } }, OmicronZoneConfig { id: f7332e25-0a75-4959-b498-faa862ee2c5f, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External } } } }, OmicronZoneConfig { id: d6729570-1e52-4d84-af84-f4f23623dfb6, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External } } } }, OmicronZoneConfig { id: 1d816ece-0442-4979-9d32-7665ef85afde, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External } } } }, OmicronZoneConfig { id: 98f9db1c-1595-4f22-a928-321363864abe, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External } } } }, OmicronZoneConfig { id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }, OmicronZoneConfig { id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:05.201540626Z","hostname":"buskin","pid":1671,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:299"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:05.20288599Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 6322d35a-04fa-4d40-ba3c-fb0208e2f2f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:05.20289741Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 719635e7-d366-4895-9272-9ade9dc77ba8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:05.225177375Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 719635e7-d366-4895-9272-9ade9dc77ba8, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:05.225262117Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ae10e87-0030-442b-9981-0b4c70a9c369, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:05.245481703Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 2ae10e87-0030-442b-9981-0b4c70a9c369, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: InternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:05.245543275Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:05.264174079Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 38d5c382-d544-48a5-9e2e-b01d5cfb5bc9, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:05.264225271Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bf26395e-9671-4bee-b96d-1c6f76e85a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:05.282098469Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: bf26395e-9671-4bee-b96d-1c6f76e85a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:05.282164821Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:05.301351813Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 426e6991-0acc-4834-a7bc-68fb4dfbb775, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:05.301425386Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 21544c6f-59af-4a59-a720-d40b64e76f5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:05.320782593Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 21544c6f-59af-4a59-a720-d40b64e76f5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:05.320845455Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ddb8e0b6-c193-419e-b893-def90acf2c6e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:05.339012094Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: ddb8e0b6-c193-419e-b893-def90acf2c6e, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: CockroachDb }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:05.339074146Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a702a2df-8aec-4a4e-9585-52df0fbccf45, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:05.35650401Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: a702a2df-8aec-4a4e-9585-52df0fbccf45, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:05.356561452Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:05.374647338Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: ExternalDns }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:05.37470508Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:05.395228016Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9882b086-6ec2-441a-b8c4-df78d6d0a302, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Clickhouse }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:05.395293368Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:05.413500527Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 4b8884ec-5901-4d3f-83a3-b1b1f952dc17, dataset_name: DatasetName { pool_name: ZpoolName { id: 14b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:05.41356835Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:05.431756539Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9f6066d0-74ff-4236-abb7-8522ff3656f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 24b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:05.431822601Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:05.45001101Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 7df09570-afa6-4fc7-ae24-8a2ea0b32af7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31bd71cd-4736-4a12-a387-9b74b050396f, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:05.450067582Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7332e25-0a75-4959-b498-faa862ee2c5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:05.468330833Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: f7332e25-0a75-4959-b498-faa862ee2c5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 616b26df-e62a-4c68-b506-f4a923d8aaf7, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:05.468392185Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6729570-1e52-4d84-af84-f4f23623dfb6, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:05.486682388Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: d6729570-1e52-4d84-af84-f4f23623dfb6, dataset_name: DatasetName { pool_name: ZpoolName { id: cd70d7f6-2354-4bf2-8012-55bf9eaf7930, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:05.48674905Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1d816ece-0442-4979-9d32-7665ef85afde, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:05.50617307Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 1d816ece-0442-4979-9d32-7665ef85afde, dataset_name: DatasetName { pool_name: ZpoolName { id: ceb4461c-cf56-4719-ad3c-14430bfdfb60, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:05.506231142Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 98f9db1c-1595-4f22-a928-321363864abe, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:05.524562225Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 98f9db1c-1595-4f22-a928-321363864abe, dataset_name: DatasetName { pool_name: ZpoolName { id: d462a7f7-b628-40fe-80ff-4e4189e2d62b, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:05.524626487Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:05.54260842Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23, dataset_name: DatasetName { pool_name: ZpoolName { id: e4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:05.542666082Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:05.560792339Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"add_dataset: NewFilesystemRequest { dataset_id: 9bfdfe18-7777-415f-abb1-32f1b43e3b55, dataset_name: DatasetName { pool_name: ZpoolName { id: f4b4dc87-ab46-49fb-a4b4-d361ae214c03, kind: External }, kind: Crucible }, responder: Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:05.563169097Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:565"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:05.579317159Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:05.579407252Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:05.579713072Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:05.579860377Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:05.676768729Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:254"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:05.679787888Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:254"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:05.679913092Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:05.738555934Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:05.738630136Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:05.738644267Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:05.738668108Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:05.738673428Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:05.738682308Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:05.738688758Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:05.900969173Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:254"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:05.90119757Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:05.946264165Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:254"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:05.946597326Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:05.946667258Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:05.946693009Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:05.947947327Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:254"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:05.961480336Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:05.961573299Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Configuring new Omicron zone: oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:05.962014394Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:05.966719108Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:05.971452844Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:05.976141859Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:05.980757901Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:05.985638042Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:05.991008549Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:05.996170459Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.114055521Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.12008971Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.124912649Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.129039805Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.133682148Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.138503437Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.143045866Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.147827654Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Configuring new Omicron zone: oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.153303154Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.158451984Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.163319914Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.168772254Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.174684389Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.180592443Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.187072186Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.194128229Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.199676912Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.205649458Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.2122225Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.217716706Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.262219172Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:271","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.308096163Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:271","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.354652376Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:271","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.386437123Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:271","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.418086035Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:271","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.419508722Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:296"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.419522793Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:188","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.473256263Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte1\", _ip: 172.30.1.6, mac: MacAddr6([168, 64, 37, 255, 215, 88]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 }, vnic: \"vopte1\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.473338405Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1268","zone_type":"external_dns"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.476946794Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:296"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.476960294Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:188","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.531559693Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte2\", _ip: 172.30.2.7, mac: MacAddr6([168, 64, 37, 255, 157, 15]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte2\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.531637155Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1268","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.53573028Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:296"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.535744701Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:188","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.592750208Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte3\", _ip: 172.30.2.5, mac: MacAddr6([168, 64, 37, 255, 247, 233]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte3\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.592823721Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1268","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.595668155Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:296"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.595681805Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:188","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.651731211Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte4\", _ip: 172.30.1.5, mac: MacAddr6([168, 64, 37, 255, 138, 152]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 }, vnic: \"vopte4\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.651803273Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1268","zone_type":"external_dns"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.653926883Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:296"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.653957434Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:188","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.712030357Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte5\", _ip: 172.30.2.6, mac: MacAddr6([168, 64, 37, 255, 232, 169]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte5\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.712101699Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1268","zone_type":"nexus"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.1.165,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.71272123Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:98"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.1.165,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.712741621Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:105"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.1.165,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.713557307Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:133"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.1.168,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.716752033Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:98"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.1.168,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.716784954Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:105"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.716797814Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.1.166,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.716838531Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:98"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.1.166,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.716845571Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:105"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.1.164,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.716916278Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:98"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.1.164,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.716923538Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:105"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.1.167,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.71696418Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:98"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.1.167,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.71696997Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:105"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.1.168,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.74581233Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:133"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.1.166,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.748362044Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:133"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.750891317Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.750913418Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.1.164,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.750929938Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:133"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.1.167,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.753094Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:133"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.755316153Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.755327733Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Configuring new Omicron zone: oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.859844046Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.865712119Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.871652945Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.87698368Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.882425194Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.892253313Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.898783978Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.905620373Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.911081433Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:06.917194454Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:20.040808676Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:20.268177894Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:20.295676758Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:20.296940519Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:22.767595895Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:22.919140322Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:23.757506881Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:23.758971389Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:23.759049272Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:23.834540621Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:23.835037838Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:23.842795873Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:23.880597934Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:23.88107418Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:23.886893311Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:23.887220022Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:23.892586838Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:23.898405409Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:23.898467431Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:23.903529708Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:23.975125309Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:23.976452623Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:23.976627469Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:24.003014065Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:24.350087556Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:24.35658487Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:24.870067926Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:24.870236612Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:25.680107625Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39\") failed: Property not found. retry in 39.952945ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:25.882931227Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c\") failed: Property not found. retry in 69.68409ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:25.893193884Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7\") failed: Property not found. retry in 35.295822ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:25.901103894Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17\") failed: Property not found. retry in 71.04303ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:25.910523213Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6\") failed: Property not found. retry in 47.012432ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:25.918866607Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2\") failed: Property not found. retry in 69.833362ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:25.926445696Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203\") failed: Property not found. retry in 37.596132ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:25.939903158Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde\") failed: Property not found. retry in 41.498594ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:25.968760616Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23\") failed: Property not found. retry in 74.344899ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:25.981745043Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55\") failed: Property not found. retry in 56.711998ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:25.991234645Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe\") failed: Property not found. retry in 48.522789ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:26.0020538Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401\") failed: Property not found. retry in 58.17121ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:26.01331286Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39\") failed: Property not found. retry in 138.757536ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:26.055529857Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c\") failed: Property not found. retry in 134.795664ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:26.063972934Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:26.114322338Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893\") failed: Property not found. retry in 32.789033ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:26.148520301Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7\") failed: Property not found. retry in 51.295223ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:26.159637996Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17\") failed: Property not found. retry in 91.449242ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:26.169311694Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6\") failed: Property not found. retry in 132.946266ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:26.178357361Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2\") failed: Property not found. retry in 79.910798ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:26.188328179Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203\") failed: Property not found. retry in 96.483491ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:26.197671086Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde\") failed: Property not found. retry in 53.521242ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:26.207384625Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23\") failed: Property not found. retry in 99.116671ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:26.216787403Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:26.217467696Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55\") failed: Property not found. retry in 59.010694ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:26.231514157Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe\") failed: Property not found. retry in 132.69026ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:26.243922285Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f\") failed: Property not found. retry in 59.381749ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:26.254402689Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45\") failed: Property not found. retry in 51.925914ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:26.264897884Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235\") failed: Property not found. retry in 52.899785ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:26.275489342Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401\") failed: Property not found. retry in 85.447862ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:26.285295564Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39\") failed: Property not found. retry in 286.553662ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:26.295796989Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c\") failed: Property not found. retry in 153.775794ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:26.305088454Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893\") failed: Property not found. retry in 126.695102ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:26.315535657Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7\") failed: Property not found. retry in 141.228216ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:26.326974473Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17\") failed: Property not found. retry in 268.937867ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:26.336049861Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6\") failed: Property not found. retry in 168.993041ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:26.344613952Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2\") failed: Property not found. retry in 239.980771ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:26.357307069Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203\") failed: Property not found. retry in 166.060506ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:26.367310208Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde\") failed: Property not found. retry in 249.339468ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:26.378387952Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23\") failed: Property not found. retry in 220.904856ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:26.390114187Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55\") failed: Property not found. retry in 252.627824ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:26.401998177Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe\") failed: Property not found. retry in 168.793369ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:26.412579005Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f\") failed: Property not found. retry in 95.390702ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:26.422589863Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45\") failed: Property not found. retry in 95.068971ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:26.438254588Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235\") failed: Property not found. retry in 123.392994ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:26.448795354Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401\") failed: Property not found. retry in 263.970867ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:26.458491473Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893\") failed: Property not found. retry in 131.634132ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:26.469146363Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7\") failed: Property not found. retry in 342.573067ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:26.481599032Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df\") failed: Property not found. retry in 41.685976ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:26.490350339Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c\") failed: Property not found. retry in 288.792985ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:26.567680609Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575\") failed: Property not found. retry in 73.576182ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:26.576710166Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6\") failed: Property not found. retry in 337.140066ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:26.586424165Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203\") failed: Property not found. retry in 462.967092ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:26.693658847Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde\") failed: Property not found. retry in 454.311582ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:26.706503169Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23\") failed: Property not found. retry in 385.00765ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:26.719573919Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df\") failed: Property not found. retry in 113.557322ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:26.731311304Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55\") failed: Property not found. retry in 351.575782ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:26.741776898Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe\") failed: Property not found. retry in 489.937235ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:26.751932362Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f\") failed: Property not found. retry in 123.062223ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:26.764114942Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45\") failed: Property not found. retry in 152.23619ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:26.775982552Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235\") failed: Property not found. retry in 296.157932ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:26.79023279Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401\") failed: Property not found. retry in 530.166721ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:26.801452738Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39\") failed: Property not found. retry in 359.434033ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:26.812051787Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575\") failed: Property not found. retry in 139.120248ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:26.825763667Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893\") failed: Property not found. retry in 406.225107ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:26.838832226Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7\") failed: Property not found. retry in 499.502285ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:26.850247281Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17\") failed: Property not found. retry in 510.776413ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:26.860413435Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2\") failed: Property not found. retry in 525.444968ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:26.870179306Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df\") failed: Property not found. retry in 297.687845ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:26.880250607Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c\") failed: Property not found. retry in 877.999815ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:26.889871212Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f\") failed: Property not found. retry in 245.632922ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:26.902915251Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6\") failed: Property not found. retry in 615.843378ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:26.938332974Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45\") failed: Property not found. retry in 508.455554ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:26.952918374Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575\") failed: Property not found. retry in 148.076576ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:26.979356232Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235\") failed: Property not found. retry in 302.494055ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:27.104078529Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55\") failed: Property not found. retry in 1.097260327s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:27.114574133Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23\") failed: Property not found. retry in 452.335577ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:27.126406062Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575\") failed: Property not found. retry in 247.507024ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:27.14368575Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f\") failed: Property not found. retry in 850.160445ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:27.166176499Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203\") failed: Property not found. retry in 732.738588ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:27.182332559Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde\") failed: Property not found. retry in 819.37059ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:27.195125719Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39\") failed: Property not found. retry in 638.040349ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:27.208169998Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df\") failed: Property not found. retry in 460.816712ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:27.219967665Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe\") failed: Property not found. retry in 705.723815ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:27.253978583Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893\") failed: Property not found. retry in 963.169145ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:27.264534649Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401\") failed: Property not found. retry in 1.120830958s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:27.343467902Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7\") failed: Property not found. retry in 1.0784592s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:27.361462163Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17\") failed: Property not found. retry in 544.80203ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:27.38116609Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575\") failed: Property not found. retry in 1.171048406s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:27.401890071Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2\") failed: Property not found. retry in 491.811611ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:27.411312971Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235\") failed: Property not found. retry in 412.234952ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:27.420486082Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45\") failed: Property not found. retry in 1.100103065s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:27.473349538Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6\") failed: Property not found. retry in 1.435478994s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:27.56687198Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23\") failed: Property not found. retry in 924.452689ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:27.591768558Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df\") failed: Property not found. retry in 842.395786ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:27.692488204Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:27.720985253Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:34181"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:27.751321649Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"0b02ec09-5ca4-44a9-8373-bc40c35800d7","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:34181","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":5555,"response_code":"200"} {"msg":"wait for service Some(\"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c\") failed: Property not found. retry in 753.996975ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:27.779984341Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235\") failed: Property not found. retry in 860.454263ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:27.934619949Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39\") failed: Property not found. retry in 887.871544ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:27.95625588Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17\") failed: Property not found. retry in 888.604277ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:28.194841927Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2\") failed: Property not found. retry in 994.868884ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:28.207443281Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203\") failed: Property not found. retry in 559.128473ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:28.219138915Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde\") failed: Property not found. retry in 1.145656551s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:28.233790936Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55\") failed: Property not found. retry in 835.919641ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:28.261706673Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe\") failed: Property not found. retry in 1.339304392s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:28.292750223Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f\") failed: Property not found. retry in 1.052271551s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:28.433389692Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893\") failed: Property not found. retry in 1.458566332s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:28.459551312Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7\") failed: Property not found. retry in 1.156009495s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:28.511101085Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401\") failed: Property not found. retry in 1.205858182s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:28.567593111Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c\") failed: Property not found. retry in 785.212636ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:28.575448599Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575\") failed: Property not found. retry in 539.862081ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:28.595095004Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23\") failed: Property not found. retry in 913.060513ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:28.66073674Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df\") failed: Property not found. retry in 602.139214ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:28.746002191Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45\") failed: Property not found. retry in 1.052740138s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:28.753396364Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203\") failed: Property not found. retry in 1.297252043s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:28.821260303Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235\") failed: Property not found. retry in 773.960269ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:28.828647006Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39\") failed: Property not found. retry in 934.913903ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:28.85221203Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6\") failed: Property not found. retry in 821.664036ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:29.011198162Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17\") failed: Property not found. retry in 1.061946957s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:29.123367287Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55\") failed: Property not found. retry in 630.522073ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:29.131332458Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575\") failed: Property not found. retry in 1.397467043s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:29.157169067Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2\") failed: Property not found. retry in 1.32875711s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:29.211869134Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df\") failed: Property not found. retry in 1.171432975s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:29.357853489Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c\") failed: Property not found. retry in 505.841208ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:29.370617978Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde\") failed: Property not found. retry in 1.296135592s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:29.388736233Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f\") failed: Property not found. retry in 787.793517ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:29.496583106Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23\") failed: Property not found. retry in 1.147971086s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:29.583996837Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235\") failed: Property not found. retry in 550.06239ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:29.612722521Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe\") failed: Property not found. retry in 672.411709ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:29.641529147Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7\") failed: Property not found. retry in 1.392413635s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:29.676351651Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55\") failed: Property not found. retry in 906.269996ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:29.77068967Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401\") failed: Property not found. retry in 759.782123ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:29.782773556Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39\") failed: Property not found. retry in 1.308765951s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:29.795419822Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45\") failed: Property not found. retry in 1.206664394s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:29.814333043Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6\") failed: Property not found. retry in 1.05075308s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:29.842390715Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:29.847976238Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:29.848301169Z","hostname":"buskin","pid":1671,"zone":"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c\") failed: Property not found. retry in 984.053789ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:29.886611197Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893\") failed: Property not found. retry in 1.285953711s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:29.927423898Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302\") failed: Property not found. retry in 38.470203ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:30.038342141Z","hostname":"buskin","pid":1671,"zone":"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302\") failed: Property not found. retry in 135.925567ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:30.087713383Z","hostname":"buskin","pid":1671,"zone":"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203\") failed: Property not found. retry in 671.184063ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:30.129476365Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235\") failed: Property not found. retry in 959.567827ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:30.174331508Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17\") failed: Property not found. retry in 1.432880304s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:30.194757029Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302\") failed: Property not found. retry in 148.254774ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:30.232787198Z","hostname":"buskin","pid":1671,"zone":"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f\") failed: Property not found. retry in 1.487450316s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:30.292558592Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe\") failed: Property not found. retry in 1.405198717s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:30.321559664Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302\") failed: Property not found. retry in 224.547966ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:30.393862709Z","hostname":"buskin","pid":1671,"zone":"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df\") failed: Property not found. retry in 1.393857164s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:30.539253285Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2\") failed: Property not found. retry in 738.069528ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:30.550422158Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401\") failed: Property not found. retry in 628.67779ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:30.558544199Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575\") failed: Property not found. retry in 1.076082305s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:30.567870095Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302\") failed: Property not found. retry in 810.736534ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:30.629756508Z","hostname":"buskin","pid":1671,"zone":"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55\") failed: Property not found. retry in 1.423388394s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:30.688454896Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde\") failed: Property not found. retry in 1.416568629s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:30.698608239Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23\") failed: Property not found. retry in 1.211200394s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:30.741387695Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203\") failed: Property not found. retry in 849.13214ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:30.810862317Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c\") failed: Property not found. retry in 1.476284391s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:30.882840851Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6\") failed: Property not found. retry in 1.435531473s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:30.910566361Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45\") failed: Property not found. retry in 687.895297ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:31.0353353Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7\") failed: Property not found. retry in 1.378625022s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:31.078209998Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39\") failed: Property not found. retry in 1.03283567s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:31.115373939Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235\") failed: Property not found. retry in 1.125496443s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:31.144169375Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401\") failed: Property not found. retry in 754.982212ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:31.197248038Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893\") failed: Property not found. retry in 874.538712ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:31.222748916Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2\") failed: Property not found. retry in 1.468617724s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:31.299294158Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302\") failed: Property not found. retry in 970.09856ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:31.450798667Z","hostname":"buskin","pid":1671,"zone":"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17\") failed: Property not found. retry in 530.7327ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:31.639527516Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575\") failed: Property not found. retry in 964.449247ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:31.656512624Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203\") failed: Property not found. retry in 1.230058008s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:31.671139415Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45\") failed: Property not found. retry in 1.106934699s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:31.735190298Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe\") failed: Property not found. retry in 554.551142ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:31.746006154Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f\") failed: Property not found. retry in 925.010339ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:31.795360975Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df\") failed: Property not found. retry in 1.213290382s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:31.943566672Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401\") failed: Property not found. retry in 868.092672ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:31.961559623Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23\") failed: Property not found. retry in 1.092823057s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:31.97213278Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893\") failed: Property not found. retry in 1.109955141s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:32.107161336Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55\") failed: Property not found. retry in 668.831929ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:32.122169079Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde\") failed: Property not found. retry in 1.36261022s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:32.129797639Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39\") failed: Property not found. retry in 1.155683803s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:32.158588285Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17\") failed: Property not found. retry in 689.75949ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:32.18034781Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235\") failed: Property not found. retry in 921.073419ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:32.282173214Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe\") failed: Property not found. retry in 1.315426519s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:32.314150935Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6\") failed: Property not found. retry in 1.374676998s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:32.357793169Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c\") failed: Property not found. retry in 893.864493ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:32.374034912Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302\") failed: Property not found. retry in 1.249602645s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:32.439019497Z","hostname":"buskin","pid":1671,"zone":"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7\") failed: Property not found. retry in 1.131221777s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:32.47231133Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575\") failed: Property not found. retry in 1.087701459s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:32.636032668Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f\") failed: Property not found. retry in 1.189057807s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:32.736733876Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2\") failed: Property not found. retry in 1.47069358s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:32.784122872Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55\") failed: Property not found. retry in 1.257181994s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:32.810210209Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401\") failed: Property not found. retry in 1.229082165s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:32.852914802Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45\") failed: Property not found. retry in 1.149127287s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:32.868302247Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17\") failed: Property not found. retry in 1.226557381s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:32.886461294Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203\") failed: Property not found. retry in 819.346678ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:32.914599968Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23\") failed: Property not found. retry in 1.455049368s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:33.084165878Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df\") failed: Property not found. retry in 873.546235ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:33.175151127Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235\") failed: Property not found. retry in 715.805201ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:33.225835731Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893\") failed: Property not found. retry in 1.024055945s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:33.248577429Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c\") failed: Property not found. retry in 1.170235905s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:33.293791834Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39\") failed: Property not found. retry in 1.10267668s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:33.357999843Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde\") failed: Property not found. retry in 662.043887ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:33.505194688Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7\") failed: Property not found. retry in 760.979073ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:33.61911052Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe\") failed: Property not found. retry in 740.579317ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:33.645213787Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302\") failed: Property not found. retry in 1.195611733s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:33.702755277Z","hostname":"buskin","pid":1671,"zone":"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575\") failed: Property not found. retry in 612.301223ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:33.735655998Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6\") failed: Property not found. retry in 1.256580555s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:33.748490386Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203\") failed: Property not found. retry in 1.499833892s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:33.759506371Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f\") failed: Property not found. retry in 1.413026394s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:33.941765708Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235\") failed: Property not found. retry in 833.695771ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:33.956659317Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45\") failed: Property not found. retry in 1.050391671s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:34.030506463Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df\") failed: Property not found. retry in 1.034566392s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:34.064187429Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55\") failed: Property not found. retry in 1.497030289s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:34.081382194Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401\") failed: Property not found. retry in 1.484159711s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:34.097320037Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17\") failed: Property not found. retry in 750.308325ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:34.135008815Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde\") failed: Property not found. retry in 1.178264892s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:34.18507864Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2\") failed: Property not found. retry in 928.924183ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:34.280456083Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893\") failed: Property not found. retry in 1.226650453s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:34.29770726Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575\") failed: Property not found. retry in 1.161800759s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:34.363182621Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7\") failed: Property not found. retry in 972.046136ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:34.393577159Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe\") failed: Property not found. retry in 1.09835234s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:34.405697887Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39\") failed: Property not found. retry in 1.480867424s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:34.474194567Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c\") failed: Property not found. retry in 1.457380767s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:34.486087708Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23\") failed: Property not found. retry in 619.759299ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:34.550414051Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235\") failed: Property not found. retry in 1.018303197s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:34.799988698Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17\") failed: Property not found. retry in 1.495602195s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:34.893535151Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302\") failed: Property not found. retry in 1.444035828s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:34.907065326Z","hostname":"buskin","pid":1671,"zone":"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6\") failed: Property not found. retry in 961.07758ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:35.017828344Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45\") failed: Property not found. retry in 1.034408778s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:35.092015441Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df\") failed: Property not found. retry in 696.627374ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:35.110315232Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23\") failed: Property not found. retry in 1.196412832s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:35.181468969Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2\") failed: Property not found. retry in 705.512067ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:35.222578289Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203\") failed: Property not found. retry in 676.482261ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:35.270524964Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f\") failed: Property not found. retry in 647.177567ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:35.366490386Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7\") failed: Property not found. retry in 1.173499274s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:35.37664663Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde\") failed: Property not found. retry in 1.10367804s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:35.388061385Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe\") failed: Property not found. retry in 1.20054168s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:35.515877953Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893\") failed: Property not found. retry in 635.51088ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:35.534423513Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575\") failed: Property not found. retry in 932.198961ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:35.543926515Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55\") failed: Property not found. retry in 1.413614293s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:35.588870861Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401\") failed: Property not found. retry in 614.107542ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:35.597324319Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df\") failed: Property not found. retry in 1.202055452s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:35.821187272Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235\") failed: Property not found. retry in 677.856626ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:35.837009242Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2\") failed: Property not found. retry in 651.729647ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:35.942773996Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c\") failed: Property not found. retry in 1.212266287s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:35.957781179Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39\") failed: Property not found. retry in 711.692566ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:35.973164964Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203\") failed: Property not found. retry in 644.41193ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:35.986005286Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6\") failed: Property not found. retry in 786.591667ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:35.999985265Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f\") failed: Property not found. retry in 830.466148ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:36.028808162Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45\") failed: Property not found. retry in 832.035925ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:36.140685127Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893\") failed: Property not found. retry in 1.34881583s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:36.18309662Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401\") failed: Property not found. retry in 585.807584ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:36.224327395Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302\") failed: Property not found. retry in 913.115206ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:36.367245109Z","hostname":"buskin","pid":1671,"zone":"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23\") failed: Property not found. retry in 1.072504641s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:36.395421244Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17\") failed: Property not found. retry in 1.40010045s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:36.411031497Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575\") failed: Property not found. retry in 963.644343ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:36.491408408Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde\") failed: Property not found. retry in 575.062055ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:36.504229558Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235\") failed: Property not found. retry in 1.386418135s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:36.530573074Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7\") failed: Property not found. retry in 576.514822ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:36.563992712Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2\") failed: Property not found. retry in 1.107315527s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:36.61081518Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203\") failed: Property not found. retry in 1.277467076s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:36.644732834Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39\") failed: Property not found. retry in 1.042053039s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:36.699742981Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe\") failed: Property not found. retry in 513.600447ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:36.731146722Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6\") failed: Property not found. retry in 1.229527045s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:36.801859945Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401\") failed: Property not found. retry in 1.048298479s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:36.824650114Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f\") failed: Property not found. retry in 885.049194ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:36.871454431Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45\") failed: Property not found. retry in 816.79136ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:36.986348435Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55\") failed: Property not found. retry in 907.211985ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:37.017463447Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df\") failed: Property not found. retry in 926.72224ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:37.0385706Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde\") failed: Property not found. retry in 1.333471248s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:37.092203782Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7\") failed: Property not found. retry in 1.492417398s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:37.153425713Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c\") failed: Property not found. retry in 1.469304056s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:37.181151584Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe\") failed: Property not found. retry in 613.813952ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:37.256756697Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302\") failed: Property not found. retry in 779.181191ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:37.290605119Z","hostname":"buskin","pid":1671,"zone":"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575\") failed: Property not found. retry in 709.680666ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:37.465432231Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23\") failed: Property not found. retry in 1.071659946s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:37.480073072Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893\") failed: Property not found. retry in 629.840427ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:37.541726157Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2\") failed: Property not found. retry in 1.28315673s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:37.728782502Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39\") failed: Property not found. retry in 1.470955504s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:37.750152694Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f\") failed: Property not found. retry in 826.994807ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:37.766261403Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45\") failed: Property not found. retry in 1.470599053s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:37.813740833Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17\") failed: Property not found. retry in 895.83219ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:37.823970129Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe\") failed: Property not found. retry in 956.075715ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:37.881978284Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401\") failed: Property not found. retry in 824.983326ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:37.891066203Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235\") failed: Property not found. retry in 1.178215253s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:37.926026081Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203\") failed: Property not found. retry in 1.445591577s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:37.952274623Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55\") failed: Property not found. retry in 1.033749981s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:37.961676032Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df\") failed: Property not found. retry in 1.403063493s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:37.977377207Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6\") failed: Property not found. retry in 1.257204021s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:38.039998895Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302\") failed: Property not found. retry in 940.828652ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:38.079156881Z","hostname":"buskin","pid":1671,"zone":"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893\") failed: Property not found. retry in 581.457789ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:38.180320974Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575\") failed: Property not found. retry in 790.962008ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:38.187572822Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde\") failed: Property not found. retry in 546.316439ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:38.437025896Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23\") failed: Property not found. retry in 1.327033737s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:38.562704004Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f\") failed: Property not found. retry in 906.195874ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:38.602371027Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7\") failed: Property not found. retry in 626.834912ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:38.655930987Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c\") failed: Property not found. retry in 980.620469ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:38.66455686Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401\") failed: Property not found. retry in 731.031087ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:38.724401766Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17\") failed: Property not found. retry in 777.076212ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:38.731926583Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893\") failed: Property not found. retry in 740.954818ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:38.770905573Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe\") failed: Property not found. retry in 982.157291ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:38.846794156Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575\") failed: Property not found. retry in 812.725741ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:38.989168433Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde\") failed: Property not found. retry in 909.225693ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:38.996579346Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55\") failed: Property not found. retry in 1.047653201s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:39.005725156Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2\") failed: Property not found. retry in 1.401631046s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:39.020988118Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302\") failed: Property not found. retry in 582.821412ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:39.029507118Z","hostname":"buskin","pid":1671,"zone":"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235\") failed: Property not found. retry in 1.292033079s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:39.113033261Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39\") failed: Property not found. retry in 1.355840933s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:39.232517376Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7\") failed: Property not found. retry in 1.157256771s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:39.292200136Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45\") failed: Property not found. retry in 796.968665ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:39.301415319Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6\") failed: Property not found. retry in 1.240768406s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:39.309230716Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df\") failed: Property not found. retry in 1.417960466s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:39.389692829Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203\") failed: Property not found. retry in 1.122210792s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:39.407555765Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401\") failed: Property not found. retry in 1.275393987s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:39.464416649Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17\") failed: Property not found. retry in 916.059909ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:39.517918591Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f\") failed: Property not found. retry in 1.17826309s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:39.525842091Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893\") failed: Property not found. retry in 1.118826042s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:39.533239814Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302\") failed: Property not found. retry in 831.798738ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:39.623065524Z","hostname":"buskin","pid":1671,"zone":"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c\") failed: Property not found. retry in 1.370835215s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:39.652926845Z","hostname":"buskin","pid":1671,"zone":"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575\") failed: Property not found. retry in 854.404309ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:39.811786623Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe\") failed: Property not found. retry in 587.145058ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:39.838923365Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde\") failed: Property not found. retry in 1.41882615s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:39.916533224Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55\") failed: Property not found. retry in 1.449038472s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:40.062997735Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45\") failed: Property not found. retry in 579.521347ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:40.108241401Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235\") failed: Property not found. retry in 927.713701ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:40.415434952Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2\") failed: Property not found. retry in 1.459836532s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:40.433848517Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe\") failed: Property not found. retry in 948.564664ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:40.442987267Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17\") failed: Property not found. retry in 1.219791974s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:40.451076173Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7\") failed: Property not found. retry in 993.275181ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:40.459122537Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302\") failed: Property not found. retry in 768.976337ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:40.466801179Z","hostname":"buskin","pid":1671,"zone":"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203\") failed: Property not found. retry in 858.329727ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:40.539517858Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6\") failed: Property not found. retry in 1.224536399s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:58:40.559837335Z","hostname":"buskin","pid":1671,"zone":"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring address fd00:1122:3344:101::9 exists","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:40.638090165Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1900"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::9, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:40.63821551Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"Ensuring address fd00:1122:3344:101::9 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:41.201855534Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1908"} {"msg":"Zone using sled underlay as gateway","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:41.201929166Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1927"} {"msg":"Setting up external-dns service","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:45.402526664Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2084"} {"msg":"Ensuring address for OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:45.402583216Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:653"} {"msg":"Ensuring address fd00:1122:3344:101::c exists","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:50.845803991Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1900"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::c, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:50.845890303Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"Ensuring address fd00:1122:3344:101::c exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:51.397952377Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1908"} {"msg":"Zone using sled underlay as gateway","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:51.404154161Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1927"} {"msg":"Setting up Nexus service","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:51.650292206Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1971"} {"msg":"Ensuring address for OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:51.650351438Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:653"} {"msg":"Ensuring address fd00:1122:3344:101::a exists","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:56.885953832Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1900"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::a, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:56.886003813Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"Ensuring address fd00:1122:3344:101::a exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:57.445799181Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1908"} {"msg":"Zone using sled underlay as gateway","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:57.445876204Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1927"} {"msg":"Setting up Nexus service","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:57.676269532Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1971"} {"msg":"Ensuring address for OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:57.676322643Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:653"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:57.791498397Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:55117"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:58:57.791745395Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"3fc0901a-6c22-4df2-b3b4-39ae15d6d5fa","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:55117","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":80,"response_code":"200"} {"msg":"Ensuring address fd00:1122:3344:101::8 exists","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:03.089701617Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1900"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::8, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:03.089762159Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"Ensuring address fd00:1122:3344:101::8 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:03.643481388Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1908"} {"msg":"Zone using sled underlay as gateway","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:03.64355255Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1927"} {"msg":"Setting up external-dns service","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:03.837440679Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2084"} {"msg":"Ensuring address for OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:03.8374922Z","hostname":"buskin","pid":1671,"zone":"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:653"} {"msg":"Ensuring address fd00:1122:3344:101::b exists","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:10.031444424Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1900"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::b, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:10.031508136Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"failed to notify nexus about a new pool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:59:10.584447149Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"220.47217818s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/ec2916b4-23be-4c03-a34b-23d4a0915a29/zpools/f4b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:59:10.585150162Z","hostname":"buskin","pid":1671,"sled_id":"ec2916b4-23be-4c03-a34b-23d4a0915a29","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1199","retry_after":"16.996935956s","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"Ensuring address fd00:1122:3344:101::b exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:10.585782493Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1908"} {"msg":"Zone using sled underlay as gateway","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:10.585798174Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1927"} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: ec2916b4-23be-4c03-a34b-23d4a0915a29, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:59:10.588832553Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"220.480026808s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:59:10.590632162Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"220.479954605s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/ec2916b4-23be-4c03-a34b-23d4a0915a29/zpools/24b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:59:10.590755176Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"220.479628254s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/ec2916b4-23be-4c03-a34b-23d4a0915a29/zpools/b462a7f7-b628-40fe-80ff-4e4189e2d62b): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"Setting up Nexus service","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:10.795390548Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:1971"} {"msg":"Ensuring address for OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:10.79543725Z","hostname":"buskin","pid":1671,"zone":"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:653"} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: ec2916b4-23be-4c03-a34b-23d4a0915a29, variant: M2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:59:10.897730089Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"220.79096853s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:59:11.267415103Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"221.156009801s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/ec2916b4-23be-4c03-a34b-23d4a0915a29/zpools/cd70d7f6-2354-4bf2-8012-55bf9eaf7930): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:59:11.356860451Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"221.246566496s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/ec2916b4-23be-4c03-a34b-23d4a0915a29/zpools/14b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:59:12.146696584Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"222.034831499s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/ec2916b4-23be-4c03-a34b-23d4a0915a29/zpools/d462a7f7-b628-40fe-80ff-4e4189e2d62b): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about sled agent","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:59:12.216458866Z","hostname":"buskin","pid":1671,"sled_id":"ec2916b4-23be-4c03-a34b-23d4a0915a29","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:741","total duration":"211.834367457s","error":"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/ec2916b4-23be-4c03-a34b-23d4a0915a29): error trying to connect: tcp connect error: Connection refused (os error 146)"} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: ec2916b4-23be-4c03-a34b-23d4a0915a29, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:59:12.226464474Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"222.116144949s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: ec2916b4-23be-4c03-a34b-23d4a0915a29, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:59:12.226598539Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"222.12163949s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:59:12.228079987Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"222.117133851s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/ec2916b4-23be-4c03-a34b-23d4a0915a29/zpools/616b26df-e62a-4c68-b506-f4a923d8aaf7): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_31bd71cd-4736-4a12-a387-9b74b050396f on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:59:12.22816108Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"222.117490093s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/ec2916b4-23be-4c03-a34b-23d4a0915a29/zpools/31bd71cd-4736-4a12-a387-9b74b050396f): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:59:12.228240553Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"222.11679974s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/ec2916b4-23be-4c03-a34b-23d4a0915a29/zpools/ceb4461c-cf56-4719-ad3c-14430bfdfb60): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", sled_id: ec2916b4-23be-4c03-a34b-23d4a0915a29, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:59:12.228412478Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"222.121812585s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: ec2916b4-23be-4c03-a34b-23d4a0915a29, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:59:12.228633896Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"222.118667662s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", sled_id: ec2916b4-23be-4c03-a34b-23d4a0915a29, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:59:12.228825582Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"222.122325642s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: ec2916b4-23be-4c03-a34b-23d4a0915a29, variant: M2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:59:12.228996768Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"222.121893298s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:59:13.276963201Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"223.164997032s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/ec2916b4-23be-4c03-a34b-23d4a0915a29/zpools/e4b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:59:13.38348057Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"223.272979129s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/ec2916b4-23be-4c03-a34b-23d4a0915a29/zpools/24b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: ec2916b4-23be-4c03-a34b-23d4a0915a29, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:59:13.5466816Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"223.436939795s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:59:13.686324897Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"223.5753191s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/ec2916b4-23be-4c03-a34b-23d4a0915a29/zpools/a462a7f7-b628-40fe-80ff-4e4189e2d62b): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:59:13.791510632Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"223.679655328s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/ec2916b4-23be-4c03-a34b-23d4a0915a29/zpools/f4b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: ec2916b4-23be-4c03-a34b-23d4a0915a29, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:59:13.897295887Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"223.792320658s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:59:14.701243615Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"224.590056692s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/ec2916b4-23be-4c03-a34b-23d4a0915a29/zpools/b462a7f7-b628-40fe-80ff-4e4189e2d62b): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:59:14.718521222Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"224.60764924s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/ec2916b4-23be-4c03-a34b-23d4a0915a29/zpools/616b26df-e62a-4c68-b506-f4a923d8aaf7): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:14.981620924Z","hostname":"buskin","pid":1671,"sled_id":"ec2916b4-23be-4c03-a34b-23d4a0915a29","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"ec2916b4-23be-4c03-a34b-23d4a0915a29"} {"msg":"failed to notify nexus about sled agent","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:59:14.983176545Z","hostname":"buskin","pid":1671,"sled_id":"ec2916b4-23be-4c03-a34b-23d4a0915a29","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:741","total duration":"214.601066627s","error":"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/ec2916b4-23be-4c03-a34b-23d4a0915a29): error trying to connect: tcp connect error: Connection refused (os error 146)"} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: ec2916b4-23be-4c03-a34b-23d4a0915a29, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:59:15.560179218Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"225.453834546s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: ec2916b4-23be-4c03-a34b-23d4a0915a29, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:59:16.069411885Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"225.959389232s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:16.137664387Z","hostname":"buskin","pid":1671,"sled_id":"ec2916b4-23be-4c03-a34b-23d4a0915a29","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"ec2916b4-23be-4c03-a34b-23d4a0915a29"} {"msg":"failed to notify nexus about sled agent","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:59:16.139524428Z","hostname":"buskin","pid":1671,"sled_id":"ec2916b4-23be-4c03-a34b-23d4a0915a29","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:741","total duration":"215.75740145s","error":"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/ec2916b4-23be-4c03-a34b-23d4a0915a29): error trying to connect: tcp connect error: Connection refused (os error 146)"} {"msg":"failed to notify nexus about a new pool oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:59:16.213439356Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"226.101816751s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/ec2916b4-23be-4c03-a34b-23d4a0915a29/zpools/d462a7f7-b628-40fe-80ff-4e4189e2d62b): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:59:16.271609377Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"226.160106735s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/ec2916b4-23be-4c03-a34b-23d4a0915a29/zpools/ceb4461c-cf56-4719-ad3c-14430bfdfb60): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: ec2916b4-23be-4c03-a34b-23d4a0915a29, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:59:16.278112231Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"226.16787602s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: ec2916b4-23be-4c03-a34b-23d4a0915a29, variant: M2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:59:16.406238389Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"226.299179693s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_31bd71cd-4736-4a12-a387-9b74b050396f on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:59:16.411960277Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"226.301232311s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/ec2916b4-23be-4c03-a34b-23d4a0915a29/zpools/31bd71cd-4736-4a12-a387-9b74b050396f): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about a new pool oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:59:16.498065716Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"226.386172101s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/ec2916b4-23be-4c03-a34b-23d4a0915a29/zpools/f4b4dc87-ab46-49fb-a4b4-d361ae214c03): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: ec2916b4-23be-4c03-a34b-23d4a0915a29, variant: M2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:59:16.792858679Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"226.68603798s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", sled_id: ec2916b4-23be-4c03-a34b-23d4a0915a29, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:59:16.987654957Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"226.881094538s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:17.122858379Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_external_dns_7e2aceaf-08e1-46f2-95ec-f9032200ba39"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:17.122937951Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_oximeter_07eefccc-a16d-41be-9df2-0faa4294ae1c"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:17.122946841Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_nexus_4243c086-1bed-4413-b0c7-7c802e7bc575"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:17.122955432Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_crucible_pantry_2cb47b44-e1c8-4597-82f0-7ee9d5d86893"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:17.122962962Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_crucible_7df09570-afa6-4fc7-ae24-8a2ea0b32af7"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:17.122985673Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_crucible_4b8884ec-5901-4d3f-83a3-b1b1f952dc17"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:17.122993833Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_crucible_d6729570-1e52-4d84-af84-f4f23623dfb6"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:17.123011044Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_crucible_9f6066d0-74ff-4236-abb7-8522ff3656f2"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:17.123019144Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_crucible_pantry_0e3a6bd4-4538-4ccd-a063-9c7342f19203"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:17.123038304Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_crucible_1d816ece-0442-4979-9d32-7665ef85afde"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:17.123046295Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_crucible_9677c70d-c2ae-4cdf-b7c0-3ded4a8fcd23"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:17.123052105Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_nexus_28701ada-e899-402b-a2fd-f7a3fd7c41df"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:17.123058705Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_crucible_9bfdfe18-7777-415f-abb1-32f1b43e3b55"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:17.123065215Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_crucible_98f9db1c-1595-4f22-a928-321363864abe"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:17.123070795Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_crucible_f7332e25-0a75-4959-b498-faa862ee2c5f"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:17.123078146Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_external_dns_a702a2df-8aec-4a4e-9585-52df0fbccf45"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:17.123085476Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_nexus_9363e23e-d709-4bd9-bca4-7fbe3ad55235"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:17.123228321Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_clickhouse_9882b086-6ec2-441a-b8c4-df78d6d0a302"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:17.123234711Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"sled-agent/src/services.rs:2766","zone":"oxz_crucible_pantry_a0f87398-93b9-40a8-90b6-7059441c2401"} {"msg":"Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:17.124011616Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:198"} {"msg":"Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:17.125788725Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:198"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:17.128474693Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"PUT","req_id":"295e44ef-e5df-4ea9-90c0-7a9446abd890","remote_addr":"[fd00:1122:3344:101::1]:35849","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":71963153,"response_code":"204"} {"msg":"Finished setting up services","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:17.137987696Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1059"} {"msg":"Writing ledger to /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:17.138000286Z","hostname":"buskin","pid":1671,"component":"RSS","file":"common/src/ledger.rs:198"} {"msg":"Writing ledger to /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:17.138170442Z","hostname":"buskin","pid":1671,"component":"RSS","file":"common/src/ledger.rs:198"} {"msg":"Handing off control to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:17.141024485Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:549"} {"msg":"Nexus address: [fd00:1122:3344:101::a]:12221","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:17.141036746Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:551"} {"msg":"rack_network_config: RackNetworkConfigV1 {\n bgp: [],\n infra_ip_first: 192.168.1.30,\n infra_ip_last: 192.168.1.30,\n ports: [\n PortConfigV1 {\n addresses: [\n V4(\n Ipv4Network {\n addr: 10.151.1.174,\n prefix: 24,\n },\n ),\n ],\n autoneg: false,\n bgp_peers: [],\n port: \"qsfp0\",\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Network {\n addr: 0.0.0.0,\n prefix: 0,\n },\n ),\n nexthop: 10.151.1.100,\n },\n ],\n switch: Switch0,\n uplink_port_fec: None,\n uplink_port_speed: Speed40G,\n },\n ],\n rack_subnet: Ipv6Network {\n addr: fd00:1122:3344:100::,\n prefix: 56,\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:17.181631709Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/ea034aee-f806-4033-865f-6a70bb9c7e3c/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:17.183981866Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/ea034aee-f806-4033-865f-6a70bb9c7e3c/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:17.2812101Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/ea034aee-f806-4033-865f-6a70bb9c7e3c/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:17.448742143Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", sled_id: ec2916b4-23be-4c03-a34b-23d4a0915a29, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:59:17.587664196Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"227.480973802s","err":"\"Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/physical-disk): error trying to connect: tcp connect error: Connection refused (os error 146)\""} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/ea034aee-f806-4033-865f-6a70bb9c7e3c/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:17.627217766Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/ea034aee-f806-4033-865f-6a70bb9c7e3c/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:17.718617158Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"failed to notify nexus about a new pool oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:59:17.896429728Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"227.78534405s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"edee3a9b-fd1f-41ba-a19c-d8445c1371ff\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Thu, 15 Feb 2024 05:59:17 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"edee3a9b-fd1f-41ba-a19c-d8445c1371ff\\\" }\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", sled_id: ec2916b4-23be-4c03-a34b-23d4a0915a29, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:59:17.914164161Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"227.806894049s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"cafa3e18-d542-47fe-bfa1-32a335eab2d0\\\", \\\"content-length\\\": \\\"173\\\", \\\"date\\\": \\\"Thu, 15 Feb 2024 05:59:17 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: sled with id \\\\\\\"ec2916b4-23be-4c03-a34b-23d4a0915a29\\\\\\\"\\\", request_id: \\\"cafa3e18-d542-47fe-bfa1-32a335eab2d0\\\" }\""} {"msg":"failed to notify nexus about a new pool oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:59:17.922002648Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"227.810207778s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"100972ce-9654-4300-83c7-fa13d85660de\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Thu, 15 Feb 2024 05:59:17 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"100972ce-9654-4300-83c7-fa13d85660de\\\" }\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", sled_id: ec2916b4-23be-4c03-a34b-23d4a0915a29, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:59:17.923121675Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"227.813526756s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"ae5223d7-8548-47a6-910a-6fa1b149e9fc\\\", \\\"content-length\\\": \\\"173\\\", \\\"date\\\": \\\"Thu, 15 Feb 2024 05:59:17 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: sled with id \\\\\\\"ec2916b4-23be-4c03-a34b-23d4a0915a29\\\\\\\"\\\", request_id: \\\"ae5223d7-8548-47a6-910a-6fa1b149e9fc\\\" }\""} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4f826213-25bb-4ad4-8c79-3e2f7b0fdf33\", \"content-length\": \"160\", \"date\": \"Thu, 15 Feb 2024 05:59:17 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: ip-pool with name \\\"oxide-service-pool\\\"\", request_id: \"4f826213-25bb-4ad4-8c79-3e2f7b0fdf33\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:17.986735105Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"failed to notify nexus about a new pool oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:59:18.04322614Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"227.931839133s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"9af8f141-7de0-4659-87e3-97456b44db2b\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Thu, 15 Feb 2024 05:59:18 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"9af8f141-7de0-4659-87e3-97456b44db2b\\\" }\""} {"msg":"failed to notify nexus about a new pool oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:59:18.074427735Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"227.96403618s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"4ebe5dd2-e4a7-4bbe-8b44-10e0e0d70d06\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Thu, 15 Feb 2024 05:59:18 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"4ebe5dd2-e4a7-4bbe-8b44-10e0e0d70d06\\\" }\""} {"msg":"Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"29ffa6e2-ebe5-4246-a41a-93f31b286a08\", \"content-length\": \"124\", \"date\": \"Thu, 15 Feb 2024 05:59:18 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"29ffa6e2-ebe5-4246-a41a-93f31b286a08\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:18.202714289Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"failed to notify nexus about a new pool oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:59:18.396625489Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"228.285679345s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"0b757a08-8fb2-473b-9751-ce28cbfc4d85\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Thu, 15 Feb 2024 05:59:18 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"0b757a08-8fb2-473b-9751-ce28cbfc4d85\\\" }\""} {"msg":"failed to notify nexus about Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: ec2916b4-23be-4c03-a34b-23d4a0915a29, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:59:18.495643301Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:223","total duration":"228.390584602s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"3b70bb41-7830-4b4a-b339-bc7a4d81a00f\\\", \\\"content-length\\\": \\\"173\\\", \\\"date\\\": \\\"Thu, 15 Feb 2024 05:59:18 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: sled with id \\\\\\\"ec2916b4-23be-4c03-a34b-23d4a0915a29\\\\\\\"\\\", request_id: \\\"3b70bb41-7830-4b4a-b339-bc7a4d81a00f\\\" }\""} {"msg":"Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"50754237-b7e9-437a-b4a8-f59d449ef471\", \"content-length\": \"124\", \"date\": \"Thu, 15 Feb 2024 05:59:18 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"50754237-b7e9-437a-b4a8-f59d449ef471\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:18.543335498Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:18.880495552Z","hostname":"buskin","pid":1671,"sled_id":"ec2916b4-23be-4c03-a34b-23d4a0915a29","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"ec2916b4-23be-4c03-a34b-23d4a0915a29"} {"msg":"contacting server nexus, registering sled","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:18.903943262Z","hostname":"buskin","pid":1671,"component":"HardwareMonitor","file":"sled-agent/src/sled_agent.rs:698","baseboard":"Baseboard { part_number: \"i86pc\", revision: 0, serial_number: \"buskin\" }","id":"ec2916b4-23be-4c03-a34b-23d4a0915a29"} {"msg":"failed to notify nexus about a new pool oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:59:19.184601581Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"229.073339398s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"0c5e9192-97b1-4591-a84c-4be382a01987\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Thu, 15 Feb 2024 05:59:18 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"0c5e9192-97b1-4591-a84c-4be382a01987\\\" }\""} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"32980343-666e-4237-8cc3-a88f6b2da5b5\", \"content-length\": \"174\", \"date\": \"Thu, 15 Feb 2024 05:59:19 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: zpool with id \\\"14b4dc87-ab46-49fb-a4b4-d361ae214c03\\\"\", request_id: \"32980343-666e-4237-8cc3-a88f6b2da5b5\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:19.313241896Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"failed to notify nexus about a new pool oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:59:19.40103973Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"229.290447399s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"813967d6-744e-4a26-a344-77d6ea483b1d\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Thu, 15 Feb 2024 05:59:19 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"813967d6-744e-4a26-a344-77d6ea483b1d\\\" }\""} {"msg":"failed to notify nexus about a new pool oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:59:19.774733845Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"229.663170463s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"9abb4a3d-0929-4e91-8b19-3fc6252f792e\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Thu, 15 Feb 2024 05:59:19 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"9abb4a3d-0929-4e91-8b19-3fc6252f792e\\\" }\""} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"cf0d5980-5cc6-416b-95cd-3fcefb1376e4\", \"content-length\": \"174\", \"date\": \"Thu, 15 Feb 2024 05:59:19 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: zpool with id \\\"14b4dc87-ab46-49fb-a4b4-d361ae214c03\\\"\", request_id: \"cf0d5980-5cc6-416b-95cd-3fcefb1376e4\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:20.186992057Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"feff7ea2-ebe9-492e-b15f-12c52502bec8\", \"content-length\": \"174\", \"date\": \"Thu, 15 Feb 2024 05:59:20 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: zpool with id \\\"14b4dc87-ab46-49fb-a4b4-d361ae214c03\\\"\", request_id: \"feff7ea2-ebe9-492e-b15f-12c52502bec8\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:20.998453841Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"failed to notify nexus about a new pool oxp_31bd71cd-4736-4a12-a387-9b74b050396f on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:59:21.32233807Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"231.211517773s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"27771029-c823-4d6c-99ac-984747bd4acc\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Thu, 15 Feb 2024 05:59:21 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"27771029-c823-4d6c-99ac-984747bd4acc\\\" }\""} {"msg":"failed to notify nexus about a new pool oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:59:21.877195996Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"231.766201652s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"f8d8554c-526d-4841-ba1c-985f108b7586\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Thu, 15 Feb 2024 05:59:21 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"f8d8554c-526d-4841-ba1c-985f108b7586\\\" }\""} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0509800e-1ffe-4f9e-8dc1-89215bb5e01a\", \"content-length\": \"174\", \"date\": \"Thu, 15 Feb 2024 05:59:21 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: zpool with id \\\"14b4dc87-ab46-49fb-a4b4-d361ae214c03\\\"\", request_id: \"0509800e-1ffe-4f9e-8dc1-89215bb5e01a\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:22.042075322Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"failed to notify nexus about a new pool oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:59:22.05176854Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"231.940623172s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"3b7226f1-65dd-4c4b-84f9-2134b1d539ce\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Thu, 15 Feb 2024 05:59:21 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"3b7226f1-65dd-4c4b-84f9-2134b1d539ce\\\" }\""} {"msg":"failed to notify nexus about a new pool oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930 on disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", model: \"synthetic-model\" }","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T05:59:22.33548794Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:273","total duration":"232.224023451s","err":"\"Error Response: status: 404 Not Found; headers: {\\\"content-type\\\": \\\"application/json\\\", \\\"x-request-id\\\": \\\"85f9f672-b09a-4097-a970-1930fd71ca3b\\\", \\\"content-length\\\": \\\"272\\\", \\\"date\\\": \\\"Thu, 15 Feb 2024 05:59:22 GMT\\\"}; value: Error { error_code: Some(\\\"ObjectNotFound\\\"), message: \\\"not found: physical-disk with id \\\\\\\"vendor = \\\\\\\"synthetic-vendor\\\\\\\", serial = \\\\\\\"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\\\\\\\", model = \\\\\\\"synthetic-model\\\\\\\"\\\\\\\"\\\", request_id: \\\"85f9f672-b09a-4097-a970-1930fd71ca3b\\\" }\""} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c43e5878-db8e-4e54-b748-59c21a9651d2\", \"content-length\": \"174\", \"date\": \"Thu, 15 Feb 2024 05:59:23 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: zpool with id \\\"14b4dc87-ab46-49fb-a4b4-d361ae214c03\\\"\", request_id: \"c43e5878-db8e-4e54-b748-59c21a9651d2\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:23.20409548Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: ec2916b4-23be-4c03-a34b-23d4a0915a29, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:23.218932478Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: ec2916b4-23be-4c03-a34b-23d4a0915a29, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:23.218952148Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", sled_id: ec2916b4-23be-4c03-a34b-23d4a0915a29, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:23.218957908Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", sled_id: ec2916b4-23be-4c03-a34b-23d4a0915a29, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:23.335955371Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c593ac50-f620-4466-84e6-1c346cfe5cde\", \"content-length\": \"174\", \"date\": \"Thu, 15 Feb 2024 05:59:24 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: zpool with id \\\"24b4dc87-ab46-49fb-a4b4-d361ae214c03\\\"\", request_id: \"c593ac50-f620-4466-84e6-1c346cfe5cde\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:24.490143083Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: ec2916b4-23be-4c03-a34b-23d4a0915a29, variant: M2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:24.625153598Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: ec2916b4-23be-4c03-a34b-23d4a0915a29, variant: M2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:24.625175699Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", sled_id: ec2916b4-23be-4c03-a34b-23d4a0915a29, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:24.62518929Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", sled_id: ec2916b4-23be-4c03-a34b-23d4a0915a29, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:24.6251943Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", sled_id: ec2916b4-23be-4c03-a34b-23d4a0915a29, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:24.62519872Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: ec2916b4-23be-4c03-a34b-23d4a0915a29, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:24.62520313Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: Put(PhysicalDiskPutRequest { model: \"synthetic-model\", serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", sled_id: ec2916b4-23be-4c03-a34b-23d4a0915a29, variant: U2, vendor: \"synthetic-vendor\" })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:24.62520736Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:24.625219371Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e70101c1-bbda-4948-a76a-cb9726a50e65\", \"content-length\": \"174\", \"date\": \"Thu, 15 Feb 2024 05:59:25 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: zpool with id \\\"24b4dc87-ab46-49fb-a4b4-d361ae214c03\\\"\", request_id: \"e70101c1-bbda-4948-a76a-cb9726a50e65\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:26.039473405Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:26.459052857Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0f749115-71e6-4eb9-bc68-81ddd80f5bd7\", \"content-length\": \"174\", \"date\": \"Thu, 15 Feb 2024 05:59:27 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: zpool with id \\\"31bd71cd-4736-4a12-a387-9b74b050396f\\\"\", request_id: \"0f749115-71e6-4eb9-bc68-81ddd80f5bd7\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:27.516378088Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-31bd71cd-4736-4a12-a387-9b74b050396f\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:27.706854524Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-616b26df-e62a-4c68-b506-f4a923d8aaf7\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:27.706877015Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:27.827743526Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:62540"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:27.827789277Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"d0e8e643-e20c-4daa-9cb4-706e7d9a2f74","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:62540","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":30,"response_code":"200"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"aa8eda25-4638-4f8d-a48c-a4b072afa184\", \"content-length\": \"174\", \"date\": \"Thu, 15 Feb 2024 05:59:28 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: zpool with id \\\"cd70d7f6-2354-4bf2-8012-55bf9eaf7930\\\"\", request_id: \"aa8eda25-4638-4f8d-a48c-a4b072afa184\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:28.750557188Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:30.344735191Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:30.344759202Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-cd70d7f6-2354-4bf2-8012-55bf9eaf7930\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:30.42537554Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-ceb4461c-cf56-4719-ad3c-14430bfdfb60\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:30.425389311Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:30.425396851Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:30.425401601Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"Nexus notification complete: ZpoolPutRequest { disk_model: \"synthetic-model\", disk_serial: \"synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03\", disk_vendor: \"synthetic-vendor\", size: ByteCount(20937965568) }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:30.425405901Z","hostname":"buskin","pid":1671,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:105"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:31.799028052Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:53769"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:31.804280174Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 15334e02-b404-488e-9ce7-2a50df738630, kind: Service { id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39 }, name: Name(\"external-dns-7e2aceaf-08e1-46f2-95ec-f9032200ba39\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 88])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 80b71d9a-f960-4a96-9d83-f9c1c3b2663c, kind: Service { id: a702a2df-8aec-4a4e-9585-52df0fbccf45 }, name: Name(\"external-dns-a702a2df-8aec-4a4e-9585-52df0fbccf45\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0e8e6699-5046-483c-bebf-fffb211be035, kind: Service { id: 28701ada-e899-402b-a2fd-f7a3fd7c41df }, name: Name(\"nexus-28701ada-e899-402b-a2fd-f7a3fd7c41df\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 9e4a840d-8987-4208-af8d-b4a2771e9cc4, kind: Service { id: 9363e23e-d709-4bd9-bca4-7fbe3ad55235 }, name: Name(\"nexus-9363e23e-d709-4bd9-bca4-7fbe3ad55235\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 232, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: a0dd5105-fcbd-4055-ae12-c3fb0abcf0fa, kind: Service { id: 4243c086-1bed-4413-b0c7-7c802e7bc575 }, name: Name(\"nexus-4243c086-1bed-4413-b0c7-7c802e7bc575\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 15])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:31.804330106Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:31.804350347Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:31.804358917Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:31.804364907Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:31.804371557Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:31.804378457Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:31.804401158Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"f34eb43d-043f-4d33-ab51-995e88d8bc12","remote_addr":"[fd00:1122:3344:101::a]:53769","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":5105,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a861ad04-3b7f-461d-a3b8-e393c21fdae0\", \"content-length\": \"127\", \"date\": \"Thu, 15 Feb 2024 05:59:31 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"a861ad04-3b7f-461d-a3b8-e393c21fdae0\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:31.97066485Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:32.185991663Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:55373"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:32.187096739Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"5c380b4f-16a2-4221-a91f-0fd58e92657b","remote_addr":"[fd00:1122:3344:101::a]:55373","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":1081,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:32.188611259Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:32.18896881Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:32.189426316Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:32.189728495Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"06f03bad-2bda-4427-9811-200b526c0d5a","remote_addr":"[fd00:1122:3344:101::a]:55373","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":2356,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:33.663474364Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:43854"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:33.663716443Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 15334e02-b404-488e-9ce7-2a50df738630, kind: Service { id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39 }, name: Name(\"external-dns-7e2aceaf-08e1-46f2-95ec-f9032200ba39\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 88])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 80b71d9a-f960-4a96-9d83-f9c1c3b2663c, kind: Service { id: a702a2df-8aec-4a4e-9585-52df0fbccf45 }, name: Name(\"external-dns-a702a2df-8aec-4a4e-9585-52df0fbccf45\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0e8e6699-5046-483c-bebf-fffb211be035, kind: Service { id: 28701ada-e899-402b-a2fd-f7a3fd7c41df }, name: Name(\"nexus-28701ada-e899-402b-a2fd-f7a3fd7c41df\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 9e4a840d-8987-4208-af8d-b4a2771e9cc4, kind: Service { id: 9363e23e-d709-4bd9-bca4-7fbe3ad55235 }, name: Name(\"nexus-9363e23e-d709-4bd9-bca4-7fbe3ad55235\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 232, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: a0dd5105-fcbd-4055-ae12-c3fb0abcf0fa, kind: Service { id: 4243c086-1bed-4413-b0c7-7c802e7bc575 }, name: Name(\"nexus-4243c086-1bed-4413-b0c7-7c802e7bc575\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 15])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:33.663762344Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:33.663792275Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:33.663804635Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:33.663814526Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:33.663824666Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:33.663858157Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:33.663878058Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"dd808588-4c2e-495a-92d3-3dc8c9c37a05","remote_addr":"[fd00:1122:3344:101::a]:43854","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":334,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5c86898d-3198-4ab2-9102-ae3aed75efe7\", \"content-length\": \"127\", \"date\": \"Thu, 15 Feb 2024 05:59:33 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"5c86898d-3198-4ab2-9102-ae3aed75efe7\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:33.735238202Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:33.797403834Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:55607"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:33.797495107Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"0c7b93ae-4f47-4400-8513-d317ad83862e","remote_addr":"[fd00:1122:3344:101::a]:55607","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":35,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:33.798141548Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:33.798168869Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:33.798384286Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:33.798614583Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"21cacf45-f198-4f01-8f74-e9e4a261bc83","remote_addr":"[fd00:1122:3344:101::a]:55607","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":956,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:37.349890624Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:48946"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:37.35005732Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 15334e02-b404-488e-9ce7-2a50df738630, kind: Service { id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39 }, name: Name(\"external-dns-7e2aceaf-08e1-46f2-95ec-f9032200ba39\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 88])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 80b71d9a-f960-4a96-9d83-f9c1c3b2663c, kind: Service { id: a702a2df-8aec-4a4e-9585-52df0fbccf45 }, name: Name(\"external-dns-a702a2df-8aec-4a4e-9585-52df0fbccf45\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0e8e6699-5046-483c-bebf-fffb211be035, kind: Service { id: 28701ada-e899-402b-a2fd-f7a3fd7c41df }, name: Name(\"nexus-28701ada-e899-402b-a2fd-f7a3fd7c41df\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 9e4a840d-8987-4208-af8d-b4a2771e9cc4, kind: Service { id: 9363e23e-d709-4bd9-bca4-7fbe3ad55235 }, name: Name(\"nexus-9363e23e-d709-4bd9-bca4-7fbe3ad55235\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 232, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: a0dd5105-fcbd-4055-ae12-c3fb0abcf0fa, kind: Service { id: 4243c086-1bed-4413-b0c7-7c802e7bc575 }, name: Name(\"nexus-4243c086-1bed-4413-b0c7-7c802e7bc575\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 15])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:37.350103471Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:37.350124122Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:37.350133142Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:37.350139352Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:37.350159763Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:37.350166493Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:37.350173403Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"9097e543-e027-4bef-af7c-52d51cfbb523","remote_addr":"[fd00:1122:3344:101::a]:48946","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":246,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ff7f82c9-01ea-4474-8e6e-50a1c8247f2b\", \"content-length\": \"127\", \"date\": \"Thu, 15 Feb 2024 05:59:37 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"ff7f82c9-01ea-4474-8e6e-50a1c8247f2b\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:37.400931291Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:37.486828322Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:58125"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:37.487420672Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"6e6716a4-0642-4619-9e02-d8fee0865bb0","remote_addr":"[fd00:1122:3344:101::a]:58125","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":569,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:37.487624129Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:37.487653459Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:37.487828865Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:37.488006101Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"a7bdd7c7-d5fc-4a78-8aff-51f71f38b6aa","remote_addr":"[fd00:1122:3344:101::a]:58125","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":403,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:41.478679777Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:37841"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:41.47937042Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 15334e02-b404-488e-9ce7-2a50df738630, kind: Service { id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39 }, name: Name(\"external-dns-7e2aceaf-08e1-46f2-95ec-f9032200ba39\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 88])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 80b71d9a-f960-4a96-9d83-f9c1c3b2663c, kind: Service { id: a702a2df-8aec-4a4e-9585-52df0fbccf45 }, name: Name(\"external-dns-a702a2df-8aec-4a4e-9585-52df0fbccf45\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0e8e6699-5046-483c-bebf-fffb211be035, kind: Service { id: 28701ada-e899-402b-a2fd-f7a3fd7c41df }, name: Name(\"nexus-28701ada-e899-402b-a2fd-f7a3fd7c41df\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 9e4a840d-8987-4208-af8d-b4a2771e9cc4, kind: Service { id: 9363e23e-d709-4bd9-bca4-7fbe3ad55235 }, name: Name(\"nexus-9363e23e-d709-4bd9-bca4-7fbe3ad55235\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 232, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: a0dd5105-fcbd-4055-ae12-c3fb0abcf0fa, kind: Service { id: 4243c086-1bed-4413-b0c7-7c802e7bc575 }, name: Name(\"nexus-4243c086-1bed-4413-b0c7-7c802e7bc575\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 15])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:41.479421101Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:41.479441332Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:41.479454562Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:41.479484413Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:41.479490154Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:41.479506174Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:41.479513374Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"bbb2641b-a6ab-44b7-bf67-91a113ebd988","remote_addr":"[fd00:1122:3344:101::a]:37841","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":785,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"78329fec-12d5-47c8-a282-aa989b782e3d\", \"content-length\": \"127\", \"date\": \"Thu, 15 Feb 2024 05:59:41 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"78329fec-12d5-47c8-a282-aa989b782e3d\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:41.532280028Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:41.622367457Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:56444"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:41.622424869Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"97905cb7-1b91-4cd3-939a-78efa546058e","remote_addr":"[fd00:1122:3344:101::a]:56444","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":35,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:41.622580464Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:41.622600255Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:41.622794241Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:41.622973827Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"6602090c-969f-4d21-a8bf-db2ed64f2306","remote_addr":"[fd00:1122:3344:101::a]:56444","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":420,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:46.920049564Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:34053"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:46.920191609Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 15334e02-b404-488e-9ce7-2a50df738630, kind: Service { id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39 }, name: Name(\"external-dns-7e2aceaf-08e1-46f2-95ec-f9032200ba39\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 88])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 80b71d9a-f960-4a96-9d83-f9c1c3b2663c, kind: Service { id: a702a2df-8aec-4a4e-9585-52df0fbccf45 }, name: Name(\"external-dns-a702a2df-8aec-4a4e-9585-52df0fbccf45\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0e8e6699-5046-483c-bebf-fffb211be035, kind: Service { id: 28701ada-e899-402b-a2fd-f7a3fd7c41df }, name: Name(\"nexus-28701ada-e899-402b-a2fd-f7a3fd7c41df\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 9e4a840d-8987-4208-af8d-b4a2771e9cc4, kind: Service { id: 9363e23e-d709-4bd9-bca4-7fbe3ad55235 }, name: Name(\"nexus-9363e23e-d709-4bd9-bca4-7fbe3ad55235\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 232, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: a0dd5105-fcbd-4055-ae12-c3fb0abcf0fa, kind: Service { id: 4243c086-1bed-4413-b0c7-7c802e7bc575 }, name: Name(\"nexus-4243c086-1bed-4413-b0c7-7c802e7bc575\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 15])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:46.9202209Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:46.920245621Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:46.920276972Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:46.920287002Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:46.920297033Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:46.920323883Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:46.920335014Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"194378d0-7466-409d-9656-258471e878c4","remote_addr":"[fd00:1122:3344:101::a]:34053","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":231,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c736921c-a893-4091-834c-19c603553529\", \"content-length\": \"127\", \"date\": \"Thu, 15 Feb 2024 05:59:46 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"c736921c-a893-4091-834c-19c603553529\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:46.971603808Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:47.06816574Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:56116"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:47.068235552Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"2888b669-97a7-4f8b-a4eb-29f92715bf4e","remote_addr":"[fd00:1122:3344:101::a]:56116","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":48,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:47.068927735Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:47.068947225Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:47.069147712Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:47.069341308Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"9126f161-3640-4f14-ae4f-2fbfdecd724c","remote_addr":"[fd00:1122:3344:101::a]:56116","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":987,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:52.151237198Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:43219"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:52.151395043Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 15334e02-b404-488e-9ce7-2a50df738630, kind: Service { id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39 }, name: Name(\"external-dns-7e2aceaf-08e1-46f2-95ec-f9032200ba39\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 88])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 80b71d9a-f960-4a96-9d83-f9c1c3b2663c, kind: Service { id: a702a2df-8aec-4a4e-9585-52df0fbccf45 }, name: Name(\"external-dns-a702a2df-8aec-4a4e-9585-52df0fbccf45\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0e8e6699-5046-483c-bebf-fffb211be035, kind: Service { id: 28701ada-e899-402b-a2fd-f7a3fd7c41df }, name: Name(\"nexus-28701ada-e899-402b-a2fd-f7a3fd7c41df\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 9e4a840d-8987-4208-af8d-b4a2771e9cc4, kind: Service { id: 9363e23e-d709-4bd9-bca4-7fbe3ad55235 }, name: Name(\"nexus-9363e23e-d709-4bd9-bca4-7fbe3ad55235\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 232, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: a0dd5105-fcbd-4055-ae12-c3fb0abcf0fa, kind: Service { id: 4243c086-1bed-4413-b0c7-7c802e7bc575 }, name: Name(\"nexus-4243c086-1bed-4413-b0c7-7c802e7bc575\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 15])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:52.151456135Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:52.151479866Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:52.151487386Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:52.151493926Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:52.151502487Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:52.151510487Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:52.151538188Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"6bf908c0-9f2e-4753-ae17-90dc13ea2625","remote_addr":"[fd00:1122:3344:101::a]:43219","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":272,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b9709793-e129-40dd-972d-b6a0b9ea786b\", \"content-length\": \"127\", \"date\": \"Thu, 15 Feb 2024 05:59:52 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"b9709793-e129-40dd-972d-b6a0b9ea786b\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:52.210341819Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:52.317971515Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:64180"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:52.318029297Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"a9b6e3ea-1ebb-4bf1-a220-be21d79c0dff","remote_addr":"[fd00:1122:3344:101::a]:64180","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":37,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:52.318178282Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:52.318197652Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:52.318380828Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:52.318538723Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"2aa90cd9-ef5f-4774-8aa4-bce2330cc04a","remote_addr":"[fd00:1122:3344:101::a]:64180","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":384,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:55.752432559Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:36252"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:55.752606315Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 15334e02-b404-488e-9ce7-2a50df738630, kind: Service { id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39 }, name: Name(\"external-dns-7e2aceaf-08e1-46f2-95ec-f9032200ba39\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 88])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 80b71d9a-f960-4a96-9d83-f9c1c3b2663c, kind: Service { id: a702a2df-8aec-4a4e-9585-52df0fbccf45 }, name: Name(\"external-dns-a702a2df-8aec-4a4e-9585-52df0fbccf45\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0e8e6699-5046-483c-bebf-fffb211be035, kind: Service { id: 28701ada-e899-402b-a2fd-f7a3fd7c41df }, name: Name(\"nexus-28701ada-e899-402b-a2fd-f7a3fd7c41df\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 9e4a840d-8987-4208-af8d-b4a2771e9cc4, kind: Service { id: 9363e23e-d709-4bd9-bca4-7fbe3ad55235 }, name: Name(\"nexus-9363e23e-d709-4bd9-bca4-7fbe3ad55235\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 232, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: a0dd5105-fcbd-4055-ae12-c3fb0abcf0fa, kind: Service { id: 4243c086-1bed-4413-b0c7-7c802e7bc575 }, name: Name(\"nexus-4243c086-1bed-4413-b0c7-7c802e7bc575\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 15])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:55.752651237Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:55.752667497Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:55.752678887Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:55.752688018Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:55.752696638Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:55.752706118Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:55.752715249Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"8cd4ecba-0484-4ea1-873f-57da615d3d42","remote_addr":"[fd00:1122:3344:101::a]:36252","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":247,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"83ee6e7f-4894-4a71-a3c4-610b7780fbd8\", \"content-length\": \"127\", \"date\": \"Thu, 15 Feb 2024 05:59:55 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"83ee6e7f-4894-4a71-a3c4-610b7780fbd8\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:55.811840121Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:55.923659134Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:58126"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:55.923722056Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"9ff85402-6a8c-4db4-a204-8c5c112651c5","remote_addr":"[fd00:1122:3344:101::a]:58126","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":38,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:55.923889911Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:55.923910732Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:55.924101929Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:55.924289815Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"ab18e240-3456-43cb-95b8-cc77d9e54a0e","remote_addr":"[fd00:1122:3344:101::a]:58126","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":420,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:57.682632153Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::d]:35010"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:57.687548244Z","hostname":"buskin","pid":1671,"uri":"/metrics/collect/ec2916b4-23be-4c03-a34b-23d4a0915a29","method":"GET","req_id":"876f643f-bb87-434d-95c9-0f9420105d36","remote_addr":"[fd00:1122:3344:101::d]:35010","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":4896,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:57.861772607Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:43886"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:57.861975494Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"146cc428-b872-43a9-8965-0ae11980b7ee","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:43886","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":25,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:59.897684013Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:61918"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:59.897855768Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 15334e02-b404-488e-9ce7-2a50df738630, kind: Service { id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39 }, name: Name(\"external-dns-7e2aceaf-08e1-46f2-95ec-f9032200ba39\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 88])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 80b71d9a-f960-4a96-9d83-f9c1c3b2663c, kind: Service { id: a702a2df-8aec-4a4e-9585-52df0fbccf45 }, name: Name(\"external-dns-a702a2df-8aec-4a4e-9585-52df0fbccf45\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0e8e6699-5046-483c-bebf-fffb211be035, kind: Service { id: 28701ada-e899-402b-a2fd-f7a3fd7c41df }, name: Name(\"nexus-28701ada-e899-402b-a2fd-f7a3fd7c41df\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 9e4a840d-8987-4208-af8d-b4a2771e9cc4, kind: Service { id: 9363e23e-d709-4bd9-bca4-7fbe3ad55235 }, name: Name(\"nexus-9363e23e-d709-4bd9-bca4-7fbe3ad55235\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 232, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: a0dd5105-fcbd-4055-ae12-c3fb0abcf0fa, kind: Service { id: 4243c086-1bed-4413-b0c7-7c802e7bc575 }, name: Name(\"nexus-4243c086-1bed-4413-b0c7-7c802e7bc575\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 15])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:59.89790149Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:59.89791499Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:59.897927921Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:59.897938171Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:59.897948311Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:59.897964102Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:59.897985733Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"039ef5fa-9c34-4e13-ba2e-5e82c5dbb0df","remote_addr":"[fd00:1122:3344:101::a]:61918","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":265,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6ec51ec3-698b-4c77-8697-2eae53f86ef3\", \"content-length\": \"127\", \"date\": \"Thu, 15 Feb 2024 05:59:59 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"6ec51ec3-698b-4c77-8697-2eae53f86ef3\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T05:59:59.953224837Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:00.045438716Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:44757"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:00.045498718Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"fa85371e-5a55-4efa-a1ab-60f7f68b8a5b","remote_addr":"[fd00:1122:3344:101::a]:44757","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":36,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:00.045674424Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:00.045693165Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:00.045868021Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:00.046040016Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"1a0817da-e7dc-4af8-80a2-0c1ffd5ef39d","remote_addr":"[fd00:1122:3344:101::a]:44757","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":390,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:11.483776342Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:44659"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:11.483929087Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 15334e02-b404-488e-9ce7-2a50df738630, kind: Service { id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39 }, name: Name(\"external-dns-7e2aceaf-08e1-46f2-95ec-f9032200ba39\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 88])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 80b71d9a-f960-4a96-9d83-f9c1c3b2663c, kind: Service { id: a702a2df-8aec-4a4e-9585-52df0fbccf45 }, name: Name(\"external-dns-a702a2df-8aec-4a4e-9585-52df0fbccf45\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0e8e6699-5046-483c-bebf-fffb211be035, kind: Service { id: 28701ada-e899-402b-a2fd-f7a3fd7c41df }, name: Name(\"nexus-28701ada-e899-402b-a2fd-f7a3fd7c41df\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 9e4a840d-8987-4208-af8d-b4a2771e9cc4, kind: Service { id: 9363e23e-d709-4bd9-bca4-7fbe3ad55235 }, name: Name(\"nexus-9363e23e-d709-4bd9-bca4-7fbe3ad55235\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 232, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: a0dd5105-fcbd-4055-ae12-c3fb0abcf0fa, kind: Service { id: 4243c086-1bed-4413-b0c7-7c802e7bc575 }, name: Name(\"nexus-4243c086-1bed-4413-b0c7-7c802e7bc575\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 15])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:11.483974159Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:11.483993129Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:11.48400148Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:11.48401929Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:11.484032181Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:11.484039061Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:11.484054191Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"4ff1f674-566e-4ac1-a1ae-067045be89cf","remote_addr":"[fd00:1122:3344:101::a]:44659","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":258,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9e223e3e-2666-40ca-98d8-f29d14678abf\", \"content-length\": \"127\", \"date\": \"Thu, 15 Feb 2024 06:00:11 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"9e223e3e-2666-40ca-98d8-f29d14678abf\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:11.546745981Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:11.706412645Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:43978"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:11.706471587Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"1b79564a-8984-401a-914b-3b03af83becb","remote_addr":"[fd00:1122:3344:101::a]:43978","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":40,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:11.706608242Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:11.706637073Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:11.70684716Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:11.707016715Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"29ac9548-cbbe-4321-ad0d-32464da94846","remote_addr":"[fd00:1122:3344:101::a]:43978","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":430,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:25.622978964Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:53670"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:25.623129529Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 15334e02-b404-488e-9ce7-2a50df738630, kind: Service { id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39 }, name: Name(\"external-dns-7e2aceaf-08e1-46f2-95ec-f9032200ba39\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 88])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 80b71d9a-f960-4a96-9d83-f9c1c3b2663c, kind: Service { id: a702a2df-8aec-4a4e-9585-52df0fbccf45 }, name: Name(\"external-dns-a702a2df-8aec-4a4e-9585-52df0fbccf45\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0e8e6699-5046-483c-bebf-fffb211be035, kind: Service { id: 28701ada-e899-402b-a2fd-f7a3fd7c41df }, name: Name(\"nexus-28701ada-e899-402b-a2fd-f7a3fd7c41df\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 9e4a840d-8987-4208-af8d-b4a2771e9cc4, kind: Service { id: 9363e23e-d709-4bd9-bca4-7fbe3ad55235 }, name: Name(\"nexus-9363e23e-d709-4bd9-bca4-7fbe3ad55235\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 232, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: a0dd5105-fcbd-4055-ae12-c3fb0abcf0fa, kind: Service { id: 4243c086-1bed-4413-b0c7-7c802e7bc575 }, name: Name(\"nexus-4243c086-1bed-4413-b0c7-7c802e7bc575\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 15])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:25.623172301Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:25.623189401Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:25.623208322Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:25.623220652Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:25.623226332Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:25.623232532Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:25.623242193Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"a9b3a409-9dd5-4e03-921e-1852edd358c4","remote_addr":"[fd00:1122:3344:101::a]:53670","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":238,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"895b6d81-08cb-42b7-bff4-37fb7fc84a97\", \"content-length\": \"127\", \"date\": \"Thu, 15 Feb 2024 06:00:25 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"895b6d81-08cb-42b7-bff4-37fb7fc84a97\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:25.683861784Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:25.79698543Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:63711"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:25.797050292Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"1f288880-0496-4f64-852c-63e1df923f73","remote_addr":"[fd00:1122:3344:101::a]:63711","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":43,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:25.797674122Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:25.797698723Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:25.7979111Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:25.798071166Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"25b2f7e4-a24d-4cdb-9e0c-b2713286f3ba","remote_addr":"[fd00:1122:3344:101::a]:63711","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":911,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:27.68262809Z","hostname":"buskin","pid":1671,"uri":"/metrics/collect/ec2916b4-23be-4c03-a34b-23d4a0915a29","method":"GET","req_id":"6710e777-a029-42a3-8356-27dce9ae002d","remote_addr":"[fd00:1122:3344:101::d]:35010","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":202,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:27.900809896Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:42101"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:27.900847787Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"d36bcfa5-4136-439a-a2c3-963e2685ab9f","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:42101","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":30,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:37.885429031Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:46780"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:37.885582626Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 15334e02-b404-488e-9ce7-2a50df738630, kind: Service { id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39 }, name: Name(\"external-dns-7e2aceaf-08e1-46f2-95ec-f9032200ba39\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 88])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 80b71d9a-f960-4a96-9d83-f9c1c3b2663c, kind: Service { id: a702a2df-8aec-4a4e-9585-52df0fbccf45 }, name: Name(\"external-dns-a702a2df-8aec-4a4e-9585-52df0fbccf45\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0e8e6699-5046-483c-bebf-fffb211be035, kind: Service { id: 28701ada-e899-402b-a2fd-f7a3fd7c41df }, name: Name(\"nexus-28701ada-e899-402b-a2fd-f7a3fd7c41df\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 9e4a840d-8987-4208-af8d-b4a2771e9cc4, kind: Service { id: 9363e23e-d709-4bd9-bca4-7fbe3ad55235 }, name: Name(\"nexus-9363e23e-d709-4bd9-bca4-7fbe3ad55235\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 232, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: a0dd5105-fcbd-4055-ae12-c3fb0abcf0fa, kind: Service { id: 4243c086-1bed-4413-b0c7-7c802e7bc575 }, name: Name(\"nexus-4243c086-1bed-4413-b0c7-7c802e7bc575\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 15])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:37.885634938Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:37.885657269Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:37.885665839Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:37.885672619Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:37.885707901Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:37.885716351Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:37.885723491Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"3bb8adae-4e2e-4ce6-9b00-5651b59e9867","remote_addr":"[fd00:1122:3344:101::a]:46780","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":259,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a60f046e-fcb1-4375-a3a1-9ce6b68414c2\", \"content-length\": \"127\", \"date\": \"Thu, 15 Feb 2024 06:00:37 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"a60f046e-fcb1-4375-a3a1-9ce6b68414c2\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:37.93621649Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:38.029101421Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:61227"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:38.029158723Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"3f7286ea-678c-44f0-9477-f00ce1ed0a5b","remote_addr":"[fd00:1122:3344:101::a]:61227","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":38,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:38.029354329Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:38.02937666Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:38.029629558Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:38.029800244Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"ea5e42ff-bfb1-46ab-8500-300a5b9e9dc3","remote_addr":"[fd00:1122:3344:101::a]:61227","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":472,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:56.265494952Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:56904"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:56.265669007Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 15334e02-b404-488e-9ce7-2a50df738630, kind: Service { id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39 }, name: Name(\"external-dns-7e2aceaf-08e1-46f2-95ec-f9032200ba39\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 88])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 80b71d9a-f960-4a96-9d83-f9c1c3b2663c, kind: Service { id: a702a2df-8aec-4a4e-9585-52df0fbccf45 }, name: Name(\"external-dns-a702a2df-8aec-4a4e-9585-52df0fbccf45\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0e8e6699-5046-483c-bebf-fffb211be035, kind: Service { id: 28701ada-e899-402b-a2fd-f7a3fd7c41df }, name: Name(\"nexus-28701ada-e899-402b-a2fd-f7a3fd7c41df\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 9e4a840d-8987-4208-af8d-b4a2771e9cc4, kind: Service { id: 9363e23e-d709-4bd9-bca4-7fbe3ad55235 }, name: Name(\"nexus-9363e23e-d709-4bd9-bca4-7fbe3ad55235\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 232, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: a0dd5105-fcbd-4055-ae12-c3fb0abcf0fa, kind: Service { id: 4243c086-1bed-4413-b0c7-7c802e7bc575 }, name: Name(\"nexus-4243c086-1bed-4413-b0c7-7c802e7bc575\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 15])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:56.265697248Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:56.265709729Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:56.265716829Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:56.265722629Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:56.26574092Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:56.26574955Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:56.265778711Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"1a07e78b-79bd-4441-980e-51cd378ac76a","remote_addr":"[fd00:1122:3344:101::a]:56904","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":252,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"41255249-9500-44ff-ac5f-099b7d4cf028\", \"content-length\": \"127\", \"date\": \"Thu, 15 Feb 2024 06:00:56 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"41255249-9500-44ff-ac5f-099b7d4cf028\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:56.318038128Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:56.412616434Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:34922"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:56.413236085Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"442891eb-dbad-4bef-ab5e-f1fe2c1f997d","remote_addr":"[fd00:1122:3344:101::a]:34922","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":583,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:56.4134035Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:56.413428861Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:56.413680749Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:56.413857955Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"0dbfaebf-efea-4b13-a74d-975845db65bb","remote_addr":"[fd00:1122:3344:101::a]:34922","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":474,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:57.682962512Z","hostname":"buskin","pid":1671,"uri":"/metrics/collect/ec2916b4-23be-4c03-a34b-23d4a0915a29","method":"GET","req_id":"ddb0b40e-547c-4082-ab9e-41a91470acdc","remote_addr":"[fd00:1122:3344:101::d]:35010","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":175,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:57.932939273Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:62582"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:00:57.932980845Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"a5b9152d-190c-413e-85d1-6ef91de600dc","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:62582","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":31,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:01:09.303065922Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:64229"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:01:09.303264808Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 15334e02-b404-488e-9ce7-2a50df738630, kind: Service { id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39 }, name: Name(\"external-dns-7e2aceaf-08e1-46f2-95ec-f9032200ba39\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 88])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 80b71d9a-f960-4a96-9d83-f9c1c3b2663c, kind: Service { id: a702a2df-8aec-4a4e-9585-52df0fbccf45 }, name: Name(\"external-dns-a702a2df-8aec-4a4e-9585-52df0fbccf45\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0e8e6699-5046-483c-bebf-fffb211be035, kind: Service { id: 28701ada-e899-402b-a2fd-f7a3fd7c41df }, name: Name(\"nexus-28701ada-e899-402b-a2fd-f7a3fd7c41df\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 9e4a840d-8987-4208-af8d-b4a2771e9cc4, kind: Service { id: 9363e23e-d709-4bd9-bca4-7fbe3ad55235 }, name: Name(\"nexus-9363e23e-d709-4bd9-bca4-7fbe3ad55235\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 232, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: a0dd5105-fcbd-4055-ae12-c3fb0abcf0fa, kind: Service { id: 4243c086-1bed-4413-b0c7-7c802e7bc575 }, name: Name(\"nexus-4243c086-1bed-4413-b0c7-7c802e7bc575\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 15])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:01:09.30332533Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:01:09.303349401Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:01:09.303361851Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:01:09.303371601Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:01:09.303400173Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:01:09.303410753Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:01:09.303420543Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"685a870c-c2ef-4d89-9b9f-08c19d9e8a05","remote_addr":"[fd00:1122:3344:101::a]:64229","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":325,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7c4142b0-f19b-49ef-a3db-948d85c677d6\", \"content-length\": \"127\", \"date\": \"Thu, 15 Feb 2024 06:01:09 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"7c4142b0-f19b-49ef-a3db-948d85c677d6\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:01:09.355643519Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:01:09.449552503Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:60288"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:01:09.449607305Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"16aa5de9-e893-4325-9f6b-0b040765bbf4","remote_addr":"[fd00:1122:3344:101::a]:60288","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":34,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:01:09.44975828Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:01:09.449787831Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:01:09.450021569Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:01:09.450206415Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"5d8af1b2-f94d-4d99-a70b-6e9248b1a90f","remote_addr":"[fd00:1122:3344:101::a]:60288","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":470,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:01:27.682896474Z","hostname":"buskin","pid":1671,"uri":"/metrics/collect/ec2916b4-23be-4c03-a34b-23d4a0915a29","method":"GET","req_id":"b031ac44-5b45-40fd-9fe6-86950a772aa3","remote_addr":"[fd00:1122:3344:101::d]:35010","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":813,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:01:27.964164123Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:58563"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:01:27.964207974Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"87dca3ce-65bb-4a03-9edd-34a4136be40a","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:58563","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":32,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:01:29.371977917Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:42474"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:01:29.372158514Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 15334e02-b404-488e-9ce7-2a50df738630, kind: Service { id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39 }, name: Name(\"external-dns-7e2aceaf-08e1-46f2-95ec-f9032200ba39\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 88])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 80b71d9a-f960-4a96-9d83-f9c1c3b2663c, kind: Service { id: a702a2df-8aec-4a4e-9585-52df0fbccf45 }, name: Name(\"external-dns-a702a2df-8aec-4a4e-9585-52df0fbccf45\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0e8e6699-5046-483c-bebf-fffb211be035, kind: Service { id: 28701ada-e899-402b-a2fd-f7a3fd7c41df }, name: Name(\"nexus-28701ada-e899-402b-a2fd-f7a3fd7c41df\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 9e4a840d-8987-4208-af8d-b4a2771e9cc4, kind: Service { id: 9363e23e-d709-4bd9-bca4-7fbe3ad55235 }, name: Name(\"nexus-9363e23e-d709-4bd9-bca4-7fbe3ad55235\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 232, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: a0dd5105-fcbd-4055-ae12-c3fb0abcf0fa, kind: Service { id: 4243c086-1bed-4413-b0c7-7c802e7bc575 }, name: Name(\"nexus-4243c086-1bed-4413-b0c7-7c802e7bc575\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 15])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:01:29.372217345Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:01:29.372253797Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:01:29.372263577Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:01:29.372270807Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:01:29.372277527Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:01:29.372284278Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:01:29.372318219Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"377932bc-0a95-4431-a007-7ed61244aaf3","remote_addr":"[fd00:1122:3344:101::a]:42474","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":320,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b2f6aba6-e988-4f6d-97dd-9fe8d82f3c06\", \"content-length\": \"127\", \"date\": \"Thu, 15 Feb 2024 06:01:29 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"b2f6aba6-e988-4f6d-97dd-9fe8d82f3c06\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:01:29.424347788Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:01:29.523283488Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:49581"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:01:29.523322609Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"48eed183-06a0-4079-9f1f-03fff019ff42","remote_addr":"[fd00:1122:3344:101::a]:49581","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":33,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:01:29.524028132Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:01:29.524046593Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:01:29.52426223Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:01:29.524424615Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"57a34b37-f1e6-4eae-9c8f-d71c95bd5072","remote_addr":"[fd00:1122:3344:101::a]:49581","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":982,"response_code":"200"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:01:40.53435726Z","hostname":"buskin","pid":1671,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:776"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:01:44.992853292Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:53366"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:01:44.992984836Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 15334e02-b404-488e-9ce7-2a50df738630, kind: Service { id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39 }, name: Name(\"external-dns-7e2aceaf-08e1-46f2-95ec-f9032200ba39\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 88])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 80b71d9a-f960-4a96-9d83-f9c1c3b2663c, kind: Service { id: a702a2df-8aec-4a4e-9585-52df0fbccf45 }, name: Name(\"external-dns-a702a2df-8aec-4a4e-9585-52df0fbccf45\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0e8e6699-5046-483c-bebf-fffb211be035, kind: Service { id: 28701ada-e899-402b-a2fd-f7a3fd7c41df }, name: Name(\"nexus-28701ada-e899-402b-a2fd-f7a3fd7c41df\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 9e4a840d-8987-4208-af8d-b4a2771e9cc4, kind: Service { id: 9363e23e-d709-4bd9-bca4-7fbe3ad55235 }, name: Name(\"nexus-9363e23e-d709-4bd9-bca4-7fbe3ad55235\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 232, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: a0dd5105-fcbd-4055-ae12-c3fb0abcf0fa, kind: Service { id: 4243c086-1bed-4413-b0c7-7c802e7bc575 }, name: Name(\"nexus-4243c086-1bed-4413-b0c7-7c802e7bc575\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 15])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:01:44.993021577Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:01:44.993041848Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:01:44.993050628Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:01:44.993056738Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:01:44.993063219Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:01:44.993069679Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:01:44.993083539Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"53d73650-2046-4487-bb4e-a50c7906d273","remote_addr":"[fd00:1122:3344:101::a]:53366","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":214,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9e8b36b6-29f4-45e6-88b4-81b98d0612ed\", \"content-length\": \"127\", \"date\": \"Thu, 15 Feb 2024 06:01:44 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"9e8b36b6-29f4-45e6-88b4-81b98d0612ed\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:01:45.055609783Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:01:45.160811159Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:47757"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:01:45.160903422Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"a74cdb6e-3df2-4773-8dc3-5c8a4159b92c","remote_addr":"[fd00:1122:3344:101::a]:47757","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":57,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:01:45.161892724Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:01:45.161918665Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:01:45.162122252Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:01:45.162280647Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"b26cd6d2-1f90-4c38-8588-cc1ad091ad24","remote_addr":"[fd00:1122:3344:101::a]:47757","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":1216,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:01:57.681754859Z","hostname":"buskin","pid":1671,"uri":"/metrics/collect/ec2916b4-23be-4c03-a34b-23d4a0915a29","method":"GET","req_id":"c1a363a0-6578-4314-94bf-ab6d59e2bf30","remote_addr":"[fd00:1122:3344:101::d]:35010","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":202,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:01:57.99625394Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:42173"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:01:57.996301502Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"214ee9a8-fc4b-4579-9cb9-3f66f32b1a43","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:42173","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":33,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:01:59.627242925Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:40247"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:01:59.627377109Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 15334e02-b404-488e-9ce7-2a50df738630, kind: Service { id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39 }, name: Name(\"external-dns-7e2aceaf-08e1-46f2-95ec-f9032200ba39\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 88])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 80b71d9a-f960-4a96-9d83-f9c1c3b2663c, kind: Service { id: a702a2df-8aec-4a4e-9585-52df0fbccf45 }, name: Name(\"external-dns-a702a2df-8aec-4a4e-9585-52df0fbccf45\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0e8e6699-5046-483c-bebf-fffb211be035, kind: Service { id: 28701ada-e899-402b-a2fd-f7a3fd7c41df }, name: Name(\"nexus-28701ada-e899-402b-a2fd-f7a3fd7c41df\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 9e4a840d-8987-4208-af8d-b4a2771e9cc4, kind: Service { id: 9363e23e-d709-4bd9-bca4-7fbe3ad55235 }, name: Name(\"nexus-9363e23e-d709-4bd9-bca4-7fbe3ad55235\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 232, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: a0dd5105-fcbd-4055-ae12-c3fb0abcf0fa, kind: Service { id: 4243c086-1bed-4413-b0c7-7c802e7bc575 }, name: Name(\"nexus-4243c086-1bed-4413-b0c7-7c802e7bc575\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 15])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:01:59.627421291Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:01:59.627439461Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:01:59.627446931Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:01:59.627462202Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:01:59.627468132Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:01:59.627476863Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:01:59.627491443Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"83a2c626-053a-4b1a-9e58-8e8d0fcf07a2","remote_addr":"[fd00:1122:3344:101::a]:40247","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":231,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f6b6bc8c-62a8-4695-aea0-fcfd0464ea70\", \"content-length\": \"127\", \"date\": \"Thu, 15 Feb 2024 06:01:59 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"f6b6bc8c-62a8-4695-aea0-fcfd0464ea70\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:01:59.681304071Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:01:59.779402393Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:62375"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:01:59.780257681Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"dfc05e76-ebc9-4344-8c78-466711a27325","remote_addr":"[fd00:1122:3344:101::a]:62375","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":825,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:01:59.780406446Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:01:59.780423756Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:01:59.780668924Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:01:59.78083748Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"0bd0ac26-1cbb-480b-97f9-37c0f465c61f","remote_addr":"[fd00:1122:3344:101::a]:62375","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":456,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:02:27.681362529Z","hostname":"buskin","pid":1671,"uri":"/metrics/collect/ec2916b4-23be-4c03-a34b-23d4a0915a29","method":"GET","req_id":"a0da64f0-75a7-4e09-9d45-aa90ae5f1fcf","remote_addr":"[fd00:1122:3344:101::d]:35010","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":193,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:02:28.029176674Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:53468"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:02:28.029915998Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"976ca2da-ffe0-48f4-b1b4-6b07f1c356bf","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:53468","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":725,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:02:36.033099286Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:64605"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:02:36.033254371Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 15334e02-b404-488e-9ce7-2a50df738630, kind: Service { id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39 }, name: Name(\"external-dns-7e2aceaf-08e1-46f2-95ec-f9032200ba39\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 88])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 80b71d9a-f960-4a96-9d83-f9c1c3b2663c, kind: Service { id: a702a2df-8aec-4a4e-9585-52df0fbccf45 }, name: Name(\"external-dns-a702a2df-8aec-4a4e-9585-52df0fbccf45\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0e8e6699-5046-483c-bebf-fffb211be035, kind: Service { id: 28701ada-e899-402b-a2fd-f7a3fd7c41df }, name: Name(\"nexus-28701ada-e899-402b-a2fd-f7a3fd7c41df\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 9e4a840d-8987-4208-af8d-b4a2771e9cc4, kind: Service { id: 9363e23e-d709-4bd9-bca4-7fbe3ad55235 }, name: Name(\"nexus-9363e23e-d709-4bd9-bca4-7fbe3ad55235\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 232, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: a0dd5105-fcbd-4055-ae12-c3fb0abcf0fa, kind: Service { id: 4243c086-1bed-4413-b0c7-7c802e7bc575 }, name: Name(\"nexus-4243c086-1bed-4413-b0c7-7c802e7bc575\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 15])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:02:36.033293472Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:02:36.033316043Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:02:36.033324893Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:02:36.033331374Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:02:36.033337574Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:02:36.033344134Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:02:36.033363495Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"730df9af-cd54-4972-9460-3de630305878","remote_addr":"[fd00:1122:3344:101::a]:64605","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":244,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a1fa4041-0fa2-4021-afbe-83dbbee60c5d\", \"content-length\": \"127\", \"date\": \"Thu, 15 Feb 2024 06:02:35 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"a1fa4041-0fa2-4021-afbe-83dbbee60c5d\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:02:36.125005205Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:02:36.239938151Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:52794"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:02:36.240041044Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"fbf18c59-b6fa-47fb-8e15-f935d3904ef3","remote_addr":"[fd00:1122:3344:101::a]:52794","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":49,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:02:36.240787548Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:02:36.240805499Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:02:36.241014936Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:02:36.241176761Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"3396ef0d-6cc5-4c88-8e24-1ac4f5167274","remote_addr":"[fd00:1122:3344:101::a]:52794","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":945,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:02:57.682282177Z","hostname":"buskin","pid":1671,"uri":"/metrics/collect/ec2916b4-23be-4c03-a34b-23d4a0915a29","method":"GET","req_id":"6f165060-b259-4789-bcd5-79e0cceb1e00","remote_addr":"[fd00:1122:3344:101::d]:35010","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":859,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:02:58.061618757Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:60574"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:02:58.061677069Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"2942999e-705f-4405-a211-e7c9205f0c20","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:60574","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":38,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:03:07.436899896Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:39868"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:03:07.437064422Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 15334e02-b404-488e-9ce7-2a50df738630, kind: Service { id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39 }, name: Name(\"external-dns-7e2aceaf-08e1-46f2-95ec-f9032200ba39\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 88])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 80b71d9a-f960-4a96-9d83-f9c1c3b2663c, kind: Service { id: a702a2df-8aec-4a4e-9585-52df0fbccf45 }, name: Name(\"external-dns-a702a2df-8aec-4a4e-9585-52df0fbccf45\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0e8e6699-5046-483c-bebf-fffb211be035, kind: Service { id: 28701ada-e899-402b-a2fd-f7a3fd7c41df }, name: Name(\"nexus-28701ada-e899-402b-a2fd-f7a3fd7c41df\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 9e4a840d-8987-4208-af8d-b4a2771e9cc4, kind: Service { id: 9363e23e-d709-4bd9-bca4-7fbe3ad55235 }, name: Name(\"nexus-9363e23e-d709-4bd9-bca4-7fbe3ad55235\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 232, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: a0dd5105-fcbd-4055-ae12-c3fb0abcf0fa, kind: Service { id: 4243c086-1bed-4413-b0c7-7c802e7bc575 }, name: Name(\"nexus-4243c086-1bed-4413-b0c7-7c802e7bc575\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 15])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:03:07.437105753Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:03:07.437138824Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:03:07.437147525Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:03:07.437152975Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:03:07.437158985Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:03:07.437175165Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:03:07.437186226Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"be58e7ad-6437-4d96-b95a-d7fbe02cfb57","remote_addr":"[fd00:1122:3344:101::a]:39868","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":264,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5a439aa5-bce8-45c3-98a1-0e459d176781\", \"content-length\": \"127\", \"date\": \"Thu, 15 Feb 2024 06:03:07 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"5a439aa5-bce8-45c3-98a1-0e459d176781\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:03:07.499347448Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:03:07.660788401Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:33571"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:03:07.660853723Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"c404567c-e2ca-4eef-99e0-1d97af55de23","remote_addr":"[fd00:1122:3344:101::a]:33571","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":41,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:03:07.661524145Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:03:07.661551366Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:03:07.661814844Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:03:07.662634231Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"63cb58d6-5c1e-4a4c-91b9-9c2f79815cac","remote_addr":"[fd00:1122:3344:101::a]:33571","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":1594,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:03:27.680931353Z","hostname":"buskin","pid":1671,"uri":"/metrics/collect/ec2916b4-23be-4c03-a34b-23d4a0915a29","method":"GET","req_id":"b2f76efc-12e2-41b2-9788-a2a262b0e078","remote_addr":"[fd00:1122:3344:101::d]:35010","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":208,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:03:28.094611682Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:62706"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:03:28.094649283Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"bf6b1baa-905b-43a4-81aa-2ef7c09bd430","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:62706","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":28,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:03:47.213890195Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:65390"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:03:47.214700121Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 15334e02-b404-488e-9ce7-2a50df738630, kind: Service { id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39 }, name: Name(\"external-dns-7e2aceaf-08e1-46f2-95ec-f9032200ba39\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 88])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 80b71d9a-f960-4a96-9d83-f9c1c3b2663c, kind: Service { id: a702a2df-8aec-4a4e-9585-52df0fbccf45 }, name: Name(\"external-dns-a702a2df-8aec-4a4e-9585-52df0fbccf45\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0e8e6699-5046-483c-bebf-fffb211be035, kind: Service { id: 28701ada-e899-402b-a2fd-f7a3fd7c41df }, name: Name(\"nexus-28701ada-e899-402b-a2fd-f7a3fd7c41df\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 9e4a840d-8987-4208-af8d-b4a2771e9cc4, kind: Service { id: 9363e23e-d709-4bd9-bca4-7fbe3ad55235 }, name: Name(\"nexus-9363e23e-d709-4bd9-bca4-7fbe3ad55235\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 232, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: a0dd5105-fcbd-4055-ae12-c3fb0abcf0fa, kind: Service { id: 4243c086-1bed-4413-b0c7-7c802e7bc575 }, name: Name(\"nexus-4243c086-1bed-4413-b0c7-7c802e7bc575\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 15])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:03:47.214740243Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:03:47.214757643Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:03:47.214766554Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:03:47.214773484Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:03:47.214780114Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:03:47.214787844Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:03:47.214803645Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"8e35715e-8bef-4dfd-995b-8940db94bfbb","remote_addr":"[fd00:1122:3344:101::a]:65390","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":897,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9fb057f1-a339-4152-9673-f744e2f5b7e9\", \"content-length\": \"127\", \"date\": \"Thu, 15 Feb 2024 06:03:46 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"9fb057f1-a339-4152-9673-f744e2f5b7e9\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:03:47.303530489Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:03:47.415120205Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:47790"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:03:47.416368126Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"64485b5a-c763-4c5b-a947-15bee0fc4292","remote_addr":"[fd00:1122:3344:101::a]:47790","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":1231,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:03:47.416509041Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:03:47.416534941Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:03:47.416741738Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:03:47.417455621Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"1568dbd5-4661-443c-b80a-17613ae4e2ff","remote_addr":"[fd00:1122:3344:101::a]:47790","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":968,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:03:57.680885156Z","hostname":"buskin","pid":1671,"uri":"/metrics/collect/ec2916b4-23be-4c03-a34b-23d4a0915a29","method":"GET","req_id":"422d6bcf-9977-4bc1-8961-82fd773f4dca","remote_addr":"[fd00:1122:3344:101::d]:35010","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":177,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:03:58.126582426Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:61466"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:03:58.127370781Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"94825073-5f52-4179-94b3-78edea42d599","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:61466","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":773,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:04:27.680989339Z","hostname":"buskin","pid":1671,"uri":"/metrics/collect/ec2916b4-23be-4c03-a34b-23d4a0915a29","method":"GET","req_id":"776c1326-1f84-487f-8161-f308367c99ac","remote_addr":"[fd00:1122:3344:101::d]:35010","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":197,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:04:28.159385754Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:58247"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:04:28.159419655Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"6a7e2e2d-2e41-4e32-9395-9a46e0d68253","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:58247","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":26,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:04:57.681177275Z","hostname":"buskin","pid":1671,"uri":"/metrics/collect/ec2916b4-23be-4c03-a34b-23d4a0915a29","method":"GET","req_id":"25dfbc7d-c8bb-4327-8315-5c70bd2637ff","remote_addr":"[fd00:1122:3344:101::d]:35010","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":763,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:04:58.192804911Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:57740"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:04:58.193327848Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"7aa1ce60-5718-46b4-9f31-e6991f65fc92","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:57740","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":506,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:05:07.10930344Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:40911"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:05:07.109447345Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 15334e02-b404-488e-9ce7-2a50df738630, kind: Service { id: 7e2aceaf-08e1-46f2-95ec-f9032200ba39 }, name: Name(\"external-dns-7e2aceaf-08e1-46f2-95ec-f9032200ba39\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 215, 88])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 80b71d9a-f960-4a96-9d83-f9c1c3b2663c, kind: Service { id: a702a2df-8aec-4a4e-9585-52df0fbccf45 }, name: Name(\"external-dns-a702a2df-8aec-4a4e-9585-52df0fbccf45\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 152])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0e8e6699-5046-483c-bebf-fffb211be035, kind: Service { id: 28701ada-e899-402b-a2fd-f7a3fd7c41df }, name: Name(\"nexus-28701ada-e899-402b-a2fd-f7a3fd7c41df\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: 9e4a840d-8987-4208-af8d-b4a2771e9cc4, kind: Service { id: 9363e23e-d709-4bd9-bca4-7fbe3ad55235 }, name: Name(\"nexus-9363e23e-d709-4bd9-bca4-7fbe3ad55235\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 232, 169])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }, NetworkInterface { id: a0dd5105-fcbd-4055-ae12-c3fb0abcf0fa, kind: Service { id: 4243c086-1bed-4413-b0c7-7c802e7bc575 }, name: Name(\"nexus-4243c086-1bed-4413-b0c7-7c802e7bc575\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 157, 15])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:05:07.109483896Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:05:07.109506186Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:05:07.109514647Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:05:07.109521657Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:05:07.109528447Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:05:07.109537367Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:05:07.109557308Z","hostname":"buskin","pid":1671,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"4c1cce78-dc1e-434b-ab6c-55c64ee93508","remote_addr":"[fd00:1122:3344:101::a]:40911","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":236,"response_code":"204"} {"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2c8a0ef5-77eb-4632-b85d-87cdcf4f137e\", \"content-length\": \"127\", \"date\": \"Thu, 15 Feb 2024 06:05:06 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"address not in lot\", request_id: \"2c8a0ef5-77eb-4632-b85d-87cdcf4f137e\" }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:05:07.164028077Z","hostname":"buskin","pid":1671,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:676"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:05:07.26303442Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:64203"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:05:07.263079541Z","hostname":"buskin","pid":1671,"uri":"/inventory","method":"GET","req_id":"9cfe1912-0a74-45d3-8548-61b58e145a9e","remote_addr":"[fd00:1122:3344:101::a]:64203","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":36,"response_code":"200"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:05:07.263228376Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Reading ledger from /pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:05:07.263247097Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"Reading ledger from /pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:05:07.263496775Z","hostname":"buskin","pid":1671,"component":"ServiceManager","file":"common/src/ledger.rs:176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:05:07.26366895Z","hostname":"buskin","pid":1671,"uri":"/omicron-zones","method":"GET","req_id":"7f91e4b0-ff88-4b05-b960-3a329c7bd0c3","remote_addr":"[fd00:1122:3344:101::a]:64203","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":462,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:05:15.790266562Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:56915"} {"msg":"ensuring instance is registered","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:05:15.798316667Z","hostname":"buskin","pid":1671,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:228","propolis_addr":"[fd00:1122:3344:101::1:0]:12400","vmm_runtime":"VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-02-15T06:05:15.489355Z }","instance_runtime":"InstanceRuntimeState { propolis_id: Some(7819f101-cc0b-4c87-9d9f-f21092e7ee65), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-02-15T06:05:15.518618249Z }","hardware":"InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: 60c1dc7b-ef63-4d65-9d02-b8683190ca41, kind: Instance { id: 741d4648-b22d-4d6c-88ec-365368fcec36 }, name: Name(\"net0\"), ip: 172.30.0.6, mac: MacAddr(MacAddr6([168, 64, 37, 249, 182, 196])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(10391284), primary: true, slot: 1 }], source_nat: SourceNatConfig { ip: 10.151.1.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.1.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 60c1dc7b-ef63-4d65-9d02-b8683190ca41, kind: Instance { id: 741d4648-b22d-4d6c-88ec-365368fcec36 }, name: Name(\"net0\"), ip: 172.30.0.6, mac: MacAddr(MacAddr6([168, 64, 37, 249, 182, 196])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(10391284), primary: true, slot: 1 }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 60c1dc7b-ef63-4d65-9d02-b8683190ca41, kind: Instance { id: 741d4648-b22d-4d6c-88ec-365368fcec36 }, name: Name(\"net0\"), ip: 172.30.0.6, mac: MacAddr(MacAddr6([168, 64, 37, 249, 182, 196])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(10391284), primary: true, slot: 1 }], filter_hosts: Some([Vpc(Vni(10391284))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 60c1dc7b-ef63-4d65-9d02-b8683190ca41, kind: Instance { id: 741d4648-b22d-4d6c-88ec-365368fcec36 }, name: Name(\"net0\"), ip: 172.30.0.6, mac: MacAddr(MacAddr6([168, 64, 37, 249, 182, 196])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(10391284), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-1661ebbb589\", read_only: false, slot: Slot(1), volume_construction_request: Volume { block_size: 512, id: 95cefd89-6915-4403-9a04-132532e08127, read_only_parent: Some(Volume { block_size: 512, id: 385f0d3a-2aa0-4b9f-b09f-46bebc22248b, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 308178f1-a663-40ac-87c3-934f17e7b365, key: Some(\"QQkOu7CoSsPfM6TRtrI0NbvJedWrkJSunfbaDkMPKkI=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::16]:19001\", \"[fd00:1122:3344:101::13]:19001\", \"[fd00:1122:3344:101::14]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 95cefd89-6915-4403-9a04-132532e08127, key: Some(\"r4htOp4JxRPV6potMHKt6NhWftifNwXkXAVMO0bYqYg=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::14]:19002\", \"[fd00:1122:3344:101::13]:19002\", \"[fd00:1122:3344:101::15]:19001\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAACvpzBPWE9YAACnME9YAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAr6cwT1hPWAAApzBPWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiI3NDFkNDY0OC1iMjJkLTRkNmMtODhlYy0zNjUzNjhmY2VjMzYiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJRzVUVDVZZjE5aXRmazBVQlltaG5ZMENFRzdPZUpTTC9ndWFGQnhXNkVLYSJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }","propolis_id":"7819f101-cc0b-4c87-9d9f-f21092e7ee65","instance_id":"741d4648-b22d-4d6c-88ec-365368fcec36"} {"msg":"registering new instance","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:05:15.79841672Z","hostname":"buskin","pid":1671,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:262","instance_id":"741d4648-b22d-4d6c-88ec-365368fcec36"} {"msg":"initializing new Instance","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:05:15.800754207Z","hostname":"buskin","pid":1671,"component":"InstanceManager","file":"sled-agent/src/instance.rs:706","state":"InstanceInitialState { hardware: InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: 60c1dc7b-ef63-4d65-9d02-b8683190ca41, kind: Instance { id: 741d4648-b22d-4d6c-88ec-365368fcec36 }, name: Name(\"net0\"), ip: 172.30.0.6, mac: MacAddr(MacAddr6([168, 64, 37, 249, 182, 196])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(10391284), primary: true, slot: 1 }], source_nat: SourceNatConfig { ip: 10.151.1.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.1.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 60c1dc7b-ef63-4d65-9d02-b8683190ca41, kind: Instance { id: 741d4648-b22d-4d6c-88ec-365368fcec36 }, name: Name(\"net0\"), ip: 172.30.0.6, mac: MacAddr(MacAddr6([168, 64, 37, 249, 182, 196])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(10391284), primary: true, slot: 1 }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 60c1dc7b-ef63-4d65-9d02-b8683190ca41, kind: Instance { id: 741d4648-b22d-4d6c-88ec-365368fcec36 }, name: Name(\"net0\"), ip: 172.30.0.6, mac: MacAddr(MacAddr6([168, 64, 37, 249, 182, 196])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(10391284), primary: true, slot: 1 }], filter_hosts: Some([Vpc(Vni(10391284))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 60c1dc7b-ef63-4d65-9d02-b8683190ca41, kind: Instance { id: 741d4648-b22d-4d6c-88ec-365368fcec36 }, name: Name(\"net0\"), ip: 172.30.0.6, mac: MacAddr(MacAddr6([168, 64, 37, 249, 182, 196])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(10391284), primary: true, slot: 1 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-1661ebbb589\", read_only: false, slot: Slot(1), volume_construction_request: Volume { block_size: 512, id: 95cefd89-6915-4403-9a04-132532e08127, read_only_parent: Some(Volume { block_size: 512, id: 385f0d3a-2aa0-4b9f-b09f-46bebc22248b, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 308178f1-a663-40ac-87c3-934f17e7b365, key: Some(\"QQkOu7CoSsPfM6TRtrI0NbvJedWrkJSunfbaDkMPKkI=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::16]:19001\", \"[fd00:1122:3344:101::13]:19001\", \"[fd00:1122:3344:101::14]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 95cefd89-6915-4403-9a04-132532e08127, key: Some(\"r4htOp4JxRPV6potMHKt6NhWftifNwXkXAVMO0bYqYg=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::14]:19002\", \"[fd00:1122:3344:101::13]:19002\", \"[fd00:1122:3344:101::15]:19001\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAACvpzBPWE9YAACnME9YAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAr6cwT1hPWAAApzBPWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiI3NDFkNDY0OC1iMjJkLTRkNmMtODhlYy0zNjUzNjhmY2VjMzYiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJRzVUVDVZZjE5aXRmazBVQlltaG5ZMENFRzdPZUpTTC9ndWFGQnhXNkVLYSJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }, instance_runtime: InstanceRuntimeState { propolis_id: Some(7819f101-cc0b-4c87-9d9f-f21092e7ee65), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-02-15T06:05:15.518618249Z }, vmm_runtime: VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-02-15T06:05:15.489355Z }, propolis_addr: [fd00:1122:3344:101::1:0]:12400 }","propolis_id":"7819f101-cc0b-4c87-9d9f-f21092e7ee65","instance_id":"741d4648-b22d-4d6c-88ec-365368fcec36"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:05:15.800824339Z","hostname":"buskin","pid":1671,"uri":"/instances/741d4648-b22d-4d6c-88ec-365368fcec36","method":"PUT","req_id":"ce42c288-95f5-4a3f-97eb-764ba6e750bf","remote_addr":"[fd00:1122:3344:101::a]:56915","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":10429,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:05:15.8985728Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:38820"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:05:15.917721149Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte6\", _ip: 172.30.0.6, mac: MacAddr6([168, 64, 37, 249, 182, 196]), slot: 1, vni: Vni { inner: 10391284 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.0.1 }, vnic: \"vopte6\" } }"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:05:15.91835647Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Configuring new Omicron zone: oxz_propolis-server_7819f101-cc0b-4c87-9d9f-f21092e7ee65","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:05:15.951827619Z","hostname":"buskin","pid":1671,"instance_id":"741d4648-b22d-4d6c-88ec-365368fcec36","component":"InstanceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_propolis-server_7819f101-cc0b-4c87-9d9f-f21092e7ee65","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:05:15.972323982Z","hostname":"buskin","pid":1671,"instance_id":"741d4648-b22d-4d6c-88ec-365368fcec36","component":"InstanceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_propolis-server_7819f101-cc0b-4c87-9d9f-f21092e7ee65:\n\n\n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:05:17.928359244Z","hostname":"buskin","pid":1671,"instance_id":"741d4648-b22d-4d6c-88ec-365368fcec36","component":"InstanceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:05:17.928556241Z","hostname":"buskin","pid":1671,"zone":"oxz_propolis-server_7819f101-cc0b-4c87-9d9f-f21092e7ee65","instance_id":"741d4648-b22d-4d6c-88ec-365368fcec36","component":"InstanceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_propolis-server_7819f101-cc0b-4c87-9d9f-f21092e7ee65\") failed: Property not found. retry in 40.097924ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T06:05:18.130814785Z","hostname":"buskin","pid":1671,"zone":"oxz_propolis-server_7819f101-cc0b-4c87-9d9f-f21092e7ee65","instance_id":"741d4648-b22d-4d6c-88ec-365368fcec36","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_7819f101-cc0b-4c87-9d9f-f21092e7ee65\") failed: Property not found. retry in 121.949313ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T06:05:18.179752842Z","hostname":"buskin","pid":1671,"zone":"oxz_propolis-server_7819f101-cc0b-4c87-9d9f-f21092e7ee65","instance_id":"741d4648-b22d-4d6c-88ec-365368fcec36","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_7819f101-cc0b-4c87-9d9f-f21092e7ee65\") failed: Property not found. retry in 163.303794ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T06:05:18.317774266Z","hostname":"buskin","pid":1671,"zone":"oxz_propolis-server_7819f101-cc0b-4c87-9d9f-f21092e7ee65","instance_id":"741d4648-b22d-4d6c-88ec-365368fcec36","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_7819f101-cc0b-4c87-9d9f-f21092e7ee65\") failed: Property not found. retry in 546.792794ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T06:05:18.496793976Z","hostname":"buskin","pid":1671,"zone":"oxz_propolis-server_7819f101-cc0b-4c87-9d9f-f21092e7ee65","instance_id":"741d4648-b22d-4d6c-88ec-365368fcec36","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_7819f101-cc0b-4c87-9d9f-f21092e7ee65\") failed: Property not found. retry in 612.524271ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T06:05:19.058084193Z","hostname":"buskin","pid":1671,"zone":"oxz_propolis-server_7819f101-cc0b-4c87-9d9f-f21092e7ee65","instance_id":"741d4648-b22d-4d6c-88ec-365368fcec36","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_7819f101-cc0b-4c87-9d9f-f21092e7ee65\") failed: Property not found. retry in 1.391971052s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T06:05:19.680013893Z","hostname":"buskin","pid":1671,"zone":"oxz_propolis-server_7819f101-cc0b-4c87-9d9f-f21092e7ee65","instance_id":"741d4648-b22d-4d6c-88ec-365368fcec36","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_7819f101-cc0b-4c87-9d9f-f21092e7ee65\") failed: Property not found. retry in 1.142473653s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T06:05:21.082816093Z","hostname":"buskin","pid":1671,"zone":"oxz_propolis-server_7819f101-cc0b-4c87-9d9f-f21092e7ee65","instance_id":"741d4648-b22d-4d6c-88ec-365368fcec36","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_7819f101-cc0b-4c87-9d9f-f21092e7ee65\") failed: Property not found. retry in 1.394387321s","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T06:05:22.236466147Z","hostname":"buskin","pid":1671,"zone":"oxz_propolis-server_7819f101-cc0b-4c87-9d9f-f21092e7ee65","instance_id":"741d4648-b22d-4d6c-88ec-365368fcec36","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Started propolis in zone: oxz_propolis-server_7819f101-cc0b-4c87-9d9f-f21092e7ee65","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:05:23.658798748Z","hostname":"buskin","pid":1671,"instance_id":"741d4648-b22d-4d6c-88ec-365368fcec36","component":"InstanceManager","file":"sled-agent/src/instance.rs:1098"} {"msg":"wait for service Some(\"oxz_propolis-server_7819f101-cc0b-4c87-9d9f-f21092e7ee65\") failed: Property not found. retry in 26.545421ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T06:05:23.666426699Z","hostname":"buskin","pid":1671,"instance_id":"741d4648-b22d-4d6c-88ec-365368fcec36","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_7819f101-cc0b-4c87-9d9f-f21092e7ee65\") failed: Property not found. retry in 52.521068ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T06:05:23.703198926Z","hostname":"buskin","pid":1671,"instance_id":"741d4648-b22d-4d6c-88ec-365368fcec36","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_7819f101-cc0b-4c87-9d9f-f21092e7ee65\") failed: Property not found. retry in 148.836218ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T06:05:23.764063076Z","hostname":"buskin","pid":1671,"instance_id":"741d4648-b22d-4d6c-88ec-365368fcec36","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_7819f101-cc0b-4c87-9d9f-f21092e7ee65\") failed: Property not found. retry in 524.33096ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T06:05:23.922766928Z","hostname":"buskin","pid":1671,"instance_id":"741d4648-b22d-4d6c-88ec-365368fcec36","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_7819f101-cc0b-4c87-9d9f-f21092e7ee65\") failed: Property not found. retry in 667.511954ms","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T06:05:24.457837224Z","hostname":"buskin","pid":1671,"instance_id":"741d4648-b22d-4d6c-88ec-365368fcec36","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Propolis SMF service is online","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:05:25.13720658Z","hostname":"buskin","pid":1671,"instance_id":"741d4648-b22d-4d6c-88ec-365368fcec36","component":"InstanceManager","file":"sled-agent/src/instance.rs:1107"} {"msg":"Propolis HTTP server online","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:05:25.194409789Z","hostname":"buskin","pid":1671,"instance_id":"741d4648-b22d-4d6c-88ec-365368fcec36","component":"InstanceManager","file":"sled-agent/src/instance.rs:1121"} {"msg":"Sending ensure request to propolis: InstanceEnsureRequest { cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAACvpzBPWE9YAACnME9YAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAr6cwT1hPWAAApzBPWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiI3NDFkNDY0OC1iMjJkLTRkNmMtODhlYy0zNjUzNjhmY2VjMzYiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJRzVUVDVZZjE5aXRmazBVQlltaG5ZMENFRzdPZUpTTC9ndWFGQnhXNkVLYSJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\"), disks: [DiskRequest { device: \"nvme\", name: \"disk-1661ebbb589\", read_only: false, slot: Slot(1), volume_construction_request: Volume { block_size: 512, id: 95cefd89-6915-4403-9a04-132532e08127, read_only_parent: Some(Volume { block_size: 512, id: 385f0d3a-2aa0-4b9f-b09f-46bebc22248b, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 308178f1-a663-40ac-87c3-934f17e7b365, key: Some(\"QQkOu7CoSsPfM6TRtrI0NbvJedWrkJSunfbaDkMPKkI=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::16]:19001\", \"[fd00:1122:3344:101::13]:19001\", \"[fd00:1122:3344:101::14]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 95cefd89-6915-4403-9a04-132532e08127, key: Some(\"r4htOp4JxRPV6potMHKt6NhWftifNwXkXAVMO0bYqYg=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::14]:19002\", \"[fd00:1122:3344:101::13]:19002\", \"[fd00:1122:3344:101::15]:19001\"] } }] } }], migrate: None, nics: [NetworkInterfaceRequest { name: \"vopte6\", slot: Slot(1) }], properties: InstanceProperties { bootrom_id: 00000000-0000-0000-0000-000000000000, description: \"Test description\", id: 741d4648-b22d-4d6c-88ec-365368fcec36, image_id: 00000000-0000-0000-0000-000000000000, memory: 1024, name: \"localshark\", vcpus: 2 } }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:05:25.19537159Z","hostname":"buskin","pid":1671,"instance_id":"741d4648-b22d-4d6c-88ec-365368fcec36","component":"InstanceManager","file":"sled-agent/src/instance.rs:453"} {"msg":"result of instance_ensure call is Ok(InstanceEnsureResponse { migrate: None })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:05:25.354142266Z","hostname":"buskin","pid":1671,"instance_id":"741d4648-b22d-4d6c-88ec-365368fcec36","component":"InstanceManager","file":"sled-agent/src/instance.rs:455"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:05:25.361677643Z","hostname":"buskin","pid":1671,"uri":"/instances/741d4648-b22d-4d6c-88ec-365368fcec36/state","method":"PUT","req_id":"45042d44-07d8-4581-89c4-56e37e4f133f","remote_addr":"[fd00:1122:3344:101::a]:38820","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":9463120,"response_code":"200"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Starting), migration_status: NoMigration, time: 2024-02-15T06:05:25.369781899Z }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:05:25.369817361Z","hostname":"buskin","pid":1671,"instance_id":"741d4648-b22d-4d6c-88ec-365368fcec36","component":"InstanceManager","file":"sled-agent/src/instance.rs:337"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:05:25.369844972Z","hostname":"buskin","pid":1671,"instance_id":"741d4648-b22d-4d6c-88ec-365368fcec36","component":"InstanceManager","file":"sled-agent/src/instance.rs:359","new_vmm_state":"VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-02-15T06:05:25.369781899Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(7819f101-cc0b-4c87-9d9f-f21092e7ee65), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-02-15T06:05:15.518618249Z }","propolis_id":"7819f101-cc0b-4c87-9d9f-f21092e7ee65"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:05:25.369862242Z","hostname":"buskin","pid":1671,"instance_id":"741d4648-b22d-4d6c-88ec-365368fcec36","component":"InstanceManager","file":"sled-agent/src/instance.rs:260","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(7819f101-cc0b-4c87-9d9f-f21092e7ee65), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-02-15T06:05:15.518618249Z }, propolis_id: 7819f101-cc0b-4c87-9d9f-f21092e7ee65, vmm_state: VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-02-15T06:05:25.369781899Z } }","instance_id":"741d4648-b22d-4d6c-88ec-365368fcec36"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Running), migration_status: NoMigration, time: 2024-02-15T06:05:25.461453391Z }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:05:25.461472491Z","hostname":"buskin","pid":1671,"instance_id":"741d4648-b22d-4d6c-88ec-365368fcec36","component":"InstanceManager","file":"sled-agent/src/instance.rs:337"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:05:25.461491442Z","hostname":"buskin","pid":1671,"instance_id":"741d4648-b22d-4d6c-88ec-365368fcec36","component":"InstanceManager","file":"sled-agent/src/instance.rs:359","new_vmm_state":"VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-02-15T06:05:25.461453391Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(7819f101-cc0b-4c87-9d9f-f21092e7ee65), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-02-15T06:05:15.518618249Z }","propolis_id":"7819f101-cc0b-4c87-9d9f-f21092e7ee65"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:05:25.461500732Z","hostname":"buskin","pid":1671,"instance_id":"741d4648-b22d-4d6c-88ec-365368fcec36","component":"InstanceManager","file":"sled-agent/src/instance.rs:260","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(7819f101-cc0b-4c87-9d9f-f21092e7ee65), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-02-15T06:05:15.518618249Z }, propolis_id: 7819f101-cc0b-4c87-9d9f-f21092e7ee65, vmm_state: VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-02-15T06:05:25.461453391Z } }","instance_id":"741d4648-b22d-4d6c-88ec-365368fcec36"} {"msg":"running automatic periodic zone bundle cleanup","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:05:27.568950237Z","hostname":"buskin","pid":1671,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:299"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:05:27.572115752Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:05:27.616115547Z","hostname":"buskin","pid":1671,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:288","time_to_next_cleanup":"599.99999576s","next_cleanup":"Instant { tv_sec: 1866, tv_nsec: 352236791 }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:05:27.681041219Z","hostname":"buskin","pid":1671,"uri":"/metrics/collect/ec2916b4-23be-4c03-a34b-23d4a0915a29","method":"GET","req_id":"2a0af7d2-5a06-467d-8d53-e0fb1523a729","remote_addr":"[fd00:1122:3344:101::d]:35010","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":804,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:05:28.227441178Z","hostname":"buskin","pid":1671,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:61972"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:05:28.22751725Z","hostname":"buskin","pid":1671,"uri":"/baseboard","method":"GET","req_id":"fc20d24f-3dde-4ba5-aea5-e7acfd39b735","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:61972","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":43,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:05:42.791012163Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:62252"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:05:42.791653884Z","hostname":"buskin","pid":1671,"uri":"/instances/741d4648-b22d-4d6c-88ec-365368fcec36/state","method":"PUT","req_id":"e923eff9-96b0-4c43-a230-9d0da0d3d29f","remote_addr":"[fd00:1122:3344:101::a]:62252","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":565,"response_code":"200"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Stopping), migration_status: NoMigration, time: 2024-02-15T06:05:42.791791898Z }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:05:42.791826389Z","hostname":"buskin","pid":1671,"instance_id":"741d4648-b22d-4d6c-88ec-365368fcec36","component":"InstanceManager","file":"sled-agent/src/instance.rs:337"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:05:42.791860331Z","hostname":"buskin","pid":1671,"instance_id":"741d4648-b22d-4d6c-88ec-365368fcec36","component":"InstanceManager","file":"sled-agent/src/instance.rs:359","new_vmm_state":"VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-02-15T06:05:42.791791898Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(7819f101-cc0b-4c87-9d9f-f21092e7ee65), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-02-15T06:05:15.518618249Z }","propolis_id":"7819f101-cc0b-4c87-9d9f-f21092e7ee65"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:05:42.791888672Z","hostname":"buskin","pid":1671,"instance_id":"741d4648-b22d-4d6c-88ec-365368fcec36","component":"InstanceManager","file":"sled-agent/src/instance.rs:260","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(7819f101-cc0b-4c87-9d9f-f21092e7ee65), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-02-15T06:05:15.518618249Z }, propolis_id: 7819f101-cc0b-4c87-9d9f-f21092e7ee65, vmm_state: VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-02-15T06:05:42.791791898Z } }","instance_id":"741d4648-b22d-4d6c-88ec-365368fcec36"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Stopped), migration_status: NoMigration, time: 2024-02-15T06:05:42.856032458Z }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:05:42.856054098Z","hostname":"buskin","pid":1671,"instance_id":"741d4648-b22d-4d6c-88ec-365368fcec36","component":"InstanceManager","file":"sled-agent/src/instance.rs:337"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:05:42.856074319Z","hostname":"buskin","pid":1671,"instance_id":"741d4648-b22d-4d6c-88ec-365368fcec36","component":"InstanceManager","file":"sled-agent/src/instance.rs:359","new_vmm_state":"VmmRuntimeState { state: Stopping, gen: Generation(6), time_updated: 2024-02-15T06:05:42.856032458Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(7819f101-cc0b-4c87-9d9f-f21092e7ee65), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-02-15T06:05:15.518618249Z }","propolis_id":"7819f101-cc0b-4c87-9d9f-f21092e7ee65"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:05:42.856087589Z","hostname":"buskin","pid":1671,"instance_id":"741d4648-b22d-4d6c-88ec-365368fcec36","component":"InstanceManager","file":"sled-agent/src/instance.rs:260","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(7819f101-cc0b-4c87-9d9f-f21092e7ee65), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-02-15T06:05:15.518618249Z }, propolis_id: 7819f101-cc0b-4c87-9d9f-f21092e7ee65, vmm_state: VmmRuntimeState { state: Stopping, gen: Generation(6), time_updated: 2024-02-15T06:05:42.856032458Z } }","instance_id":"741d4648-b22d-4d6c-88ec-365368fcec36"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Destroyed), migration_status: NoMigration, time: 2024-02-15T06:05:42.988256471Z }","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:05:42.988284512Z","hostname":"buskin","pid":1671,"instance_id":"741d4648-b22d-4d6c-88ec-365368fcec36","component":"InstanceManager","file":"sled-agent/src/instance.rs:337"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:05:42.988311303Z","hostname":"buskin","pid":1671,"instance_id":"741d4648-b22d-4d6c-88ec-365368fcec36","component":"InstanceManager","file":"sled-agent/src/instance.rs:359","new_vmm_state":"VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-02-15T06:05:42.988256471Z }","new_instance_state":"InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-02-15T06:05:42.988256471Z }","propolis_id":"7819f101-cc0b-4c87-9d9f-f21092e7ee65"} {"msg":"terminating VMM that has exited","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:05:42.988324943Z","hostname":"buskin","pid":1671,"instance_id":"741d4648-b22d-4d6c-88ec-365368fcec36","component":"InstanceManager","file":"sled-agent/src/instance.rs:374","instance_id":"741d4648-b22d-4d6c-88ec-365368fcec36"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:05:42.988370605Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"Received GetLatestResources(Sender { inner: Some(Inner { state: State { is_complete: false, is_closed: false, is_rx_task_set: true, is_tx_task_set: false } }) })","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:05:42.9888295Z","hostname":"buskin","pid":1671,"component":"StorageManager","file":"sled-storage/src/manager.rs:346"} {"msg":"creating zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:05:42.988866731Z","hostname":"buskin","pid":1671,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:436","context":"ZoneBundleContext { storage_dirs: [\"/pool/int/a462a7f7-b628-40fe-80ff-4e4189e2d62b/debug/bundle/zone\", \"/pool/int/b462a7f7-b628-40fe-80ff-4e4189e2d62b/debug/bundle/zone\"], cause: TerminatedInstance, extra_log_dirs: [\"/pool/ext/14b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/debug\", \"/pool/ext/24b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/debug\", \"/pool/ext/31bd71cd-4736-4a12-a387-9b74b050396f/crypt/debug\", \"/pool/ext/616b26df-e62a-4c68-b506-f4a923d8aaf7/crypt/debug\", \"/pool/ext/cd70d7f6-2354-4bf2-8012-55bf9eaf7930/crypt/debug\", \"/pool/ext/ceb4461c-cf56-4719-ad3c-14430bfdfb60/crypt/debug\", \"/pool/ext/d462a7f7-b628-40fe-80ff-4e4189e2d62b/crypt/debug\", \"/pool/ext/e4b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/debug\", \"/pool/ext/f4b4dc87-ab46-49fb-a4b4-d361ae214c03/crypt/debug\"] }","zone_name":"oxz_propolis-server_7819f101-cc0b-4c87-9d9f-f21092e7ee65"} {"msg":"finished zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:05:43.640643671Z","hostname":"buskin","pid":1671,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:1229","metadata":"ZoneBundleMetadata { id: ZoneBundleId { zone_name: \"oxz_propolis-server_7819f101-cc0b-4c87-9d9f-f21092e7ee65\", bundle_id: ff97d98d-c1a9-40cf-9050-9d45985a51ec }, time_created: 2024-02-15T06:05:42.989088228Z, version: 0, cause: TerminatedInstance }"} {"msg":"Halting and removing zone: oxz_propolis-server_7819f101-cc0b-4c87-9d9f-f21092e7ee65","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T06:05:43.640734084Z","hostname":"buskin","pid":1671,"instance_id":"741d4648-b22d-4d6c-88ec-365368fcec36","component":"InstanceManager","file":"sled-agent/src/instance.rs:547"} {"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:05:45.251179003Z","hostname":"buskin","pid":1671,"instance_id":"741d4648-b22d-4d6c-88ec-365368fcec36","component":"InstanceManager","file":"illumos-utils/src/zone.rs:269"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:05:45.276142483Z","hostname":"buskin","pid":1671,"instance_id":"741d4648-b22d-4d6c-88ec-365368fcec36","component":"InstanceManager","file":"sled-agent/src/instance.rs:260","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-02-15T06:05:42.988256471Z }, propolis_id: 7819f101-cc0b-4c87-9d9f-f21092e7ee65, vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-02-15T06:05:42.988256471Z } }","instance_id":"741d4648-b22d-4d6c-88ec-365368fcec36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:05:45.348513271Z","hostname":"buskin","pid":1671,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:42621"} {"msg":"unset_virtual_nic_host unimplmented","v":0,"name":"SledAgent","level":40,"time":"2024-02-15T06:05:45.351614533Z","hostname":"buskin","pid":1671,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:620"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:05:45.351632433Z","hostname":"buskin","pid":1671,"uri":"/v2p/60c1dc7b-ef63-4d65-9d02-b8683190ca41","method":"DELETE","req_id":"70d01b73-fd02-4346-b786-17cf7e8c5ffe","remote_addr":"[fd00:1122:3344:101::a]:42621","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/711a749/dropshot/src/server.rs:849","latency_us":3008,"response_code":"204"} {"msg":"State monitoring task complete","v":0,"name":"SledAgent","level":30,"time":"2024-02-15T06:05:45.511091211Z","hostname":"buskin","pid":1671,"instance_id":"741d4648-b22d-4d6c-88ec-365368fcec36","component":"InstanceManager","file":"sled-agent/src/instance.rs:490"}